Flesh out vkheader implementation, uses registry and custom callbacks to fill out...
[panamaz] / src / notzed.vkheader.test / classes / vulkan / test / TestMandelbrot.java
1  /*
2 The MIT License (MIT)
3
4 Copyright (C) 2017 Eric Arneb├Ąck
5 Copyright (C) 2019 Michael Zucchi
6
7 Permission is hereby granted, free of charge, to any person obtaining a copy
8 of this software and associated documentation files (the "Software"), to deal
9 in the Software without restriction, including without limitation the rights
10 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 copies of the Software, and to permit persons to whom the Software is
12 furnished to do so, subject to the following conditions:
13
14 The above copyright notice and this permission notice shall be included in
15 all copies or substantial portions of the Software.
16
17 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 THE SOFTWARE.
24
25  */
26
27 /*
28  * This is a Java conversion of a C conversion of this:
29  * https://github.com/Erkaman/vulkan_minimal_compute
30  *
31  * It's been simplified a bit and converted to the 'zvk' api.
32  */
33
34 package vulkan.test;
35
36 import java.io.InputStream;
37 import java.io.FileOutputStream;
38 import java.io.IOException;
39 import java.nio.channels.Channels;
40 import java.nio.ByteBuffer;
41 import java.nio.ByteOrder;
42
43 import java.awt.Graphics;
44 import java.awt.Image;
45 import java.awt.Toolkit;
46 import java.awt.event.ActionEvent;
47 import java.awt.event.KeyEvent;
48 import java.awt.image.MemoryImageSource;
49 import javax.swing.AbstractAction;
50 import javax.swing.JComponent;
51 import javax.swing.JFrame;
52 import javax.swing.JPanel;
53 import javax.swing.KeyStroke;
54
55 import java.lang.ref.WeakReference;
56
57 import java.lang.invoke.*;
58 import jdk.incubator.foreign.*;
59 import jdk.incubator.foreign.MemoryLayout.PathElement;
60 import au.notzed.nativez.*;
61
62 import vulkan.*;
63
64 import static vulkan.VkConstants.*;
65
66 public class TestMandelbrot {
67         static final boolean debug = true;
68         ResourceScope scope = ResourceScope.newSharedScope();
69
70         int WIDTH = 1920*1;
71         int HEIGHT = 1080*1;
72
73         VkInstance instance;
74         VkPhysicalDevice physicalDevice;
75
76         VkDevice device;
77         VkQueue computeQueue;
78
79         long dstBufferSize = WIDTH * HEIGHT * 4;
80         //VkBuffer dstBuffer;
81         //VkDeviceMemory dstMemory;
82         BufferMemory dst;
83
84         VkDescriptorSetLayout descriptorSetLayout;
85         VkDescriptorPool descriptorPool;
86         HandleArray<VkDescriptorSet> descriptorSets = VkDescriptorSet.createArray(1, (SegmentAllocator)scope);
87
88         int computeQueueIndex;
89         VkPhysicalDeviceMemoryProperties deviceMemoryProperties;
90
91         String mandelbrot_entry = "main";
92         IntArray mandelbrot_cs;
93
94         VkShaderModule mandelbrotShader;
95         VkPipelineLayout pipelineLayout;
96         HandleArray<VkPipeline> computePipeline = VkPipeline.createArray(1, (SegmentAllocator)scope);
97
98         VkCommandPool commandPool;
99         HandleArray<VkCommandBuffer> commandBuffers;
100
101         record BufferMemory ( VkBuffer buffer, VkDeviceMemory memory ) {};
102
103         VkDebugUtilsMessengerEXT logger;
104
105         void init_debug() throws Exception {
106                 if (!debug)
107                         return;
108                 try (Frame frame = Frame.frame()) {
109                         var cb = PFN_vkDebugUtilsMessengerCallbackEXT.upcall((severity, flags, data, dummy) -> {
110                                         System.out.printf("Debug: %d: %s\n", severity, data.getMessage());
111                                         return 0;
112                                 }, scope);
113                         VkDebugUtilsMessengerCreateInfoEXT info = VkDebugUtilsMessengerCreateInfoEXT.create(frame,
114                                 0,
115                                 VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT
116                                 | VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT
117                                 | VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
118                                 VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT | VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT,
119                                 cb,
120                                 null);
121
122                         logger = instance.vkCreateDebugUtilsMessengerEXT(info, null, scope);
123                 }
124
125                 //typedef VkBool32 (*PFN_vkDebugUtilsMessengerCallbackEXT)(VkDebugUtilsMessageSeverityFlagBitsEXT, VkDebugUtilsMessageTypeFlagsEXT, const VkDebugUtilsMessengerCallbackDataEXT *, void *);
126
127         }
128
129         void dumpStructure(GroupLayout layout, MemorySegment s) {
130                 System.out.println(s.address());
131                 for (MemoryLayout m: layout.memberLayouts()) {
132                         if (m instanceof jdk.incubator.foreign.ValueLayout) {
133                                 var vh = layout.varHandle(jdk.incubator.foreign.MemoryLayout.PathElement.groupElement(m.name().get()));
134                                 System.out.printf(" %s: %s\n", m.name().get(), vh.get(s));
135                         }
136                 }
137         }
138
139         void init_instance() throws Exception {
140                 try (Frame frame = Frame.frame()) {
141                         VkInstanceCreateInfo info = VkInstanceCreateInfo.create(frame,
142                                 0,
143                                 VkApplicationInfo.create(frame, "test", 1, "test-engine", 2, VK_MAKE_API_VERSION(0, 1, 0, 0)),
144                                 new String[] { "VK_LAYER_KHRONOS_validation" },
145                                 debug ? new String[] { "VK_EXT_debug_utils" } : null
146                                 );
147
148                         instance = VkInstance.vkCreateInstance(info, null, scope);
149                 }
150         }
151
152         void init_device() throws Exception {
153                 try (Frame frame = Frame.frame()) {
154                         HandleArray<VkPhysicalDevice> devs;
155                         int count;
156                         int res;
157
158                         devs = instance.vkEnumeratePhysicalDevices(frame, scope);
159
160                         int best = 0;
161                         int devid = -1;
162                         int queueid = -1;
163
164                         for (int i=0;i<devs.length();i++) {
165                                 VkPhysicalDevice dev = devs.getAtIndex(i);
166                                 VkQueueFamilyProperties famprops = dev.vkGetPhysicalDeviceQueueFamilyProperties(frame);
167                                 int family_count = (int)famprops.length();
168
169                                 for (int j=0;j<family_count;j++) {
170                                         int score = 0;
171
172                                         if ((famprops.getQueueFlagsAtIndex(j) & VK_QUEUE_COMPUTE_BIT) != 0)
173                                                 score += 1;
174                                         if ((famprops.getQueueFlagsAtIndex(j) & VK_QUEUE_GRAPHICS_BIT) == 0)
175                                                 score += 1;
176
177                                         if (score > best) {
178                                                 score = best;
179                                                 devid = i;
180                                                 queueid = j;
181                                         }
182                                 }
183                         }
184
185                         if (devid == -1)
186                                 throw new Exception("Cannot find a suitable device");
187
188                         computeQueueIndex = queueid;
189                         physicalDevice = devs.getAtIndex(devid);
190
191                         FloatArray qpri = FloatArray.create(frame, 0.0f);
192                         VkDeviceQueueCreateInfo qinfo = VkDeviceQueueCreateInfo.create(
193                                 frame,
194                                 0,
195                                 queueid,
196                                 qpri);
197                         VkDeviceCreateInfo devinfo = VkDeviceCreateInfo.create(
198                                 frame,
199                                 0,
200                                 qinfo,
201                                 null,
202                                 null,
203                                 null);
204
205                         device = physicalDevice.vkCreateDevice(devinfo, null, scope);
206
207                         System.out.printf("device = %s\n", device.address());
208
209                         // NOTE: app scope
210                         deviceMemoryProperties = VkPhysicalDeviceMemoryProperties.create((SegmentAllocator)scope);
211                         physicalDevice.vkGetPhysicalDeviceMemoryProperties(deviceMemoryProperties);
212
213                         computeQueue = device.vkGetDeviceQueue(queueid, 0, scope);
214                 }
215         }
216
217         /**
218          * Buffers are created in three steps:
219          * 1) create buffer, specifying usage and size
220          * 2) allocate memory based on memory requirements
221          * 3) bind memory
222          *
223          */
224         BufferMemory init_buffer(long dataSize, int usage, int properties) throws Exception {
225                 try (Frame frame = Frame.frame()) {
226                         VkMemoryRequirements req = VkMemoryRequirements.create(frame);
227                         VkBufferCreateInfo buf_info = VkBufferCreateInfo.create(frame,
228                                 0,
229                                 dataSize,
230                                 usage,
231                                 VK_SHARING_MODE_EXCLUSIVE,
232                                 null);
233
234                         VkBuffer buffer = device.vkCreateBuffer(buf_info, null, scope);
235
236                         device.vkGetBufferMemoryRequirements(buffer, req);
237
238                         VkMemoryAllocateInfo alloc = VkMemoryAllocateInfo.create(frame,
239                                 req.getSize(),
240                                 find_memory_type(deviceMemoryProperties, req.getMemoryTypeBits(), properties));
241
242                         VkDeviceMemory memory = device.vkAllocateMemory(alloc, null, scope);
243
244                         device.vkBindBufferMemory(buffer, memory, 0);
245
246                         return new BufferMemory(buffer, memory);
247                 }
248         }
249
250         /**
251          * Descriptors are used to bind and describe memory blocks
252          * to shaders.
253          *
254          * *Pool is used to allocate descriptors, it is per-device.
255          * *Layout is used to group descriptors for a given pipeline,
256          * The descriptors describe individually-addressable blocks.
257          */
258         void init_descriptor() throws Exception {
259                 try (Frame frame = Frame.frame()) {
260                         /* Create descriptorset layout */
261                         VkDescriptorSetLayoutBinding layout_binding = VkDescriptorSetLayoutBinding.create(frame,
262                                 0,
263                                 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
264                                 1,
265                                 VK_SHADER_STAGE_COMPUTE_BIT,
266                                 null);
267
268                         VkDescriptorSetLayoutCreateInfo descriptor_layout = VkDescriptorSetLayoutCreateInfo.create(frame,
269                                 0,
270                                 layout_binding);
271
272                         descriptorSetLayout = device.vkCreateDescriptorSetLayout(descriptor_layout, null, scope);
273
274                         /* Create descriptor pool */
275                         VkDescriptorPoolSize type_count = VkDescriptorPoolSize.create(frame,
276                                 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
277                                 1);
278
279                         VkDescriptorPoolCreateInfo descriptor_pool = VkDescriptorPoolCreateInfo.create(frame,
280                                 0,
281                                 1,
282                                 type_count);
283
284                         descriptorPool = device.vkCreateDescriptorPool(descriptor_pool, null, scope);
285
286                         /* Allocate from pool */
287                         HandleArray<VkDescriptorSetLayout> layout_table = VkDescriptorSetLayout.createArray(1, frame);
288
289                         layout_table.setAtIndex(0, descriptorSetLayout);
290
291                         VkDescriptorSetAllocateInfo alloc_info = VkDescriptorSetAllocateInfo.create(frame,
292                                 descriptorPool,
293                                 1,
294                                 layout_table);
295
296                         device.vkAllocateDescriptorSets(alloc_info, descriptorSets);
297
298                         /* Bind a buffer to the descriptor */
299                         VkDescriptorBufferInfo bufferInfo = VkDescriptorBufferInfo.create(frame,
300                                 dst.buffer,
301                                 0,
302                                 dstBufferSize);
303
304                         VkWriteDescriptorSet writeSet = VkWriteDescriptorSet.create(frame,
305                                 descriptorSets.getAtIndex(0),
306                                 0,
307                                 0,
308                                 VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
309                                 null,
310                                 bufferInfo,
311                                 null);
312
313                         device.vkUpdateDescriptorSets(1, writeSet, 0, null);
314                 }
315         }
316
317         /**
318          * Create the compute pipeline.  This is the shader and data layouts for it.
319          */
320         void init_pipeline() throws Exception {
321                 try (Frame frame = Frame.frame()) {
322                         /* Set shader code */
323                         VkShaderModuleCreateInfo vsInfo = VkShaderModuleCreateInfo.create(frame,
324                                 0,
325                                 mandelbrot_cs.length() * 4,
326                                 mandelbrot_cs);
327
328                         mandelbrotShader = device.vkCreateShaderModule(vsInfo, null, scope);
329
330                         /* Link shader to layout */
331                         HandleArray<VkDescriptorSetLayout> layout_table = VkDescriptorSetLayout.createArray(1, frame);
332
333                         layout_table.setAtIndex(0, descriptorSetLayout);
334
335                         VkPipelineLayoutCreateInfo pipelineinfo = VkPipelineLayoutCreateInfo.create(frame,
336                                 0,
337                                 1,
338                                 layout_table,
339                                 null);
340
341                         pipelineLayout = device.vkCreatePipelineLayout(pipelineinfo, null, scope);
342
343                         /* Create pipeline */
344                         VkComputePipelineCreateInfo pipeline = VkComputePipelineCreateInfo.create(frame,
345                                 0,
346                                 pipelineLayout,
347                                 null,
348                                 0);
349
350                         VkPipelineShaderStageCreateInfo stage = pipeline.getStage();
351
352                         stage.setStage(VK_SHADER_STAGE_COMPUTE_BIT);
353                         stage.setModule(mandelbrotShader);
354                         stage.setName(mandelbrot_entry);
355
356                         device.vkCreateComputePipelines(null, 1, pipeline, null, computePipeline);
357                 }
358         }
359
360         /**
361          * Create a command buffer, this is somewhat like a display list.
362          */
363         void init_command_buffer() throws Exception {
364                 try (Frame frame = Frame.frame()) {
365                         VkCommandPoolCreateInfo poolinfo = VkCommandPoolCreateInfo.create(frame,
366                                 0,
367                                 computeQueueIndex);
368
369                         commandPool = device.vkCreateCommandPool(poolinfo, null, scope);
370
371                         VkCommandBufferAllocateInfo cmdinfo = VkCommandBufferAllocateInfo.create(frame,
372                                 commandPool,
373                                 VK_COMMAND_BUFFER_LEVEL_PRIMARY,
374                                 1);
375
376                         // should it take a scope?
377                         commandBuffers = VkCommandBuffer.createArray(instance, 1, (SegmentAllocator)scope, scope);
378                         device.vkAllocateCommandBuffers(cmdinfo, commandBuffers);
379
380                         /* Fill command buffer with commands for later operation */
381                         VkCommandBufferBeginInfo beginInfo = VkCommandBufferBeginInfo.create(frame,
382                                 VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
383                                 null);
384
385                         commandBuffers.get(0).vkBeginCommandBuffer(beginInfo);
386
387                         /* Bind the compute operation and data */
388                         commandBuffers.get(0).vkCmdBindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, computePipeline.get(0));
389                         commandBuffers.get(0).vkCmdBindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, descriptorSets, 0, null);
390
391                         /* Run it */
392                         commandBuffers.get(0).vkCmdDispatch(WIDTH, HEIGHT, 1);
393
394                         commandBuffers.get(0).vkEndCommandBuffer();
395                 }
396         }
397
398         /**
399          * Execute the pre-created command buffer.
400          *
401          * A fence is used to wait for completion.
402          */
403         void execute() throws Exception {
404                 try (Frame frame = Frame.frame()) {
405                         VkSubmitInfo submitInfo = VkSubmitInfo.create(frame);
406
407                         submitInfo.setCommandBufferCount(1);
408                         submitInfo.setCommandBuffers(commandBuffers);
409
410                         /* Create fence to mark the task completion */
411                         VkFence fence;
412                         HandleArray<VkFence> fences = VkFence.createArray(1, frame);
413                         VkFenceCreateInfo fenceInfo = VkFenceCreateInfo.create(frame);
414
415                         // maybe this should take a HandleArray<Fence> rather than being a constructor
416                         // FIXME: some local scope
417                         fence = device.vkCreateFence(fenceInfo, null, scope);
418                         fences.set(0, fence);
419
420                         /* Await completion */
421                         computeQueue.vkQueueSubmit(1, submitInfo, fence);
422
423                         int VK_TRUE = 1;
424                         int res;
425                         do {
426                                 res = device.vkWaitForFences(1, fences, VK_TRUE, 1000000);
427                         } while (res == VK_TIMEOUT);
428
429                         device.vkDestroyFence(fence, null);
430                 }
431         }
432
433         void shutdown() {
434                 device.vkDestroyCommandPool(commandPool, null);
435                 device.vkDestroyPipeline(computePipeline.getAtIndex(0), null);
436                 device.vkDestroyPipelineLayout(pipelineLayout, null);
437                 device.vkDestroyShaderModule(mandelbrotShader, null);
438
439                 device.vkDestroyDescriptorPool(descriptorPool, null);
440                 device.vkDestroyDescriptorSetLayout(descriptorSetLayout, null);
441
442                 device.vkFreeMemory(dst.memory(), null);
443                 device.vkDestroyBuffer(dst.buffer(), null);
444
445                 device.vkDestroyDevice(null);
446                 if (logger != null)
447                         instance.vkDestroyDebugUtilsMessengerEXT(logger, null);
448                 instance.vkDestroyInstance(null);
449         }
450
451         /**
452          * Accesses the gpu buffer, converts it to RGB byte, and saves it as a pam file.
453          */
454         void save_result() throws Exception {
455                 try (ResourceScope scope = ResourceScope.newConfinedScope()) {
456                         MemorySegment mem = device.vkMapMemory(dst.memory(), 0, dstBufferSize, 0, scope);
457                         byte[] pixels = new byte[WIDTH * HEIGHT * 3];
458
459                         System.out.printf("map %d bytes\n", dstBufferSize);
460
461                         for (int i = 0; i < WIDTH * HEIGHT; i++) {
462                                 pixels[i * 3 + 0] = mem.get(Memory.BYTE, i * 4 + 0);
463                                 pixels[i * 3 + 1] = mem.get(Memory.BYTE, i * 4 + 1);
464                                 pixels[i * 3 + 2] = mem.get(Memory.BYTE, i * 4 + 2);
465                         }
466
467                         device.vkUnmapMemory(dst.memory());
468
469                         pam_save("mandelbrot.pam", WIDTH, HEIGHT, 3, pixels);
470                 }
471         }
472
473         void show_result() throws Exception {
474                 try (ResourceScope scope = ResourceScope.newConfinedScope()) {
475                         MemorySegment mem = device.vkMapMemory(dst.memory(), 0, dstBufferSize, 0, scope);
476                         int[] pixels = new int[WIDTH * HEIGHT];
477
478                         System.out.printf("map %d bytes\n", dstBufferSize);
479
480                         MemorySegment.ofArray(pixels).copyFrom(mem);
481
482                         device.vkUnmapMemory(dst.memory());
483
484                         swing_show(WIDTH, HEIGHT, pixels);
485                 }
486         }
487
488         /**
489          * Trivial pnm format image output.
490          */
491         void pam_save(String name, int width, int height, int depth, byte[] pixels) throws IOException {
492                 try (FileOutputStream fos = new FileOutputStream(name)) {
493                         fos.write(String.format("P6\n%d\n%d\n255\n", width, height).getBytes());
494                         fos.write(pixels);
495                         System.out.printf("wrote: %s\n", name);
496                 }
497         }
498
499         static class DataImage extends JPanel {
500
501                 final int w, h, stride;
502                 final MemoryImageSource source;
503                 final Image image;
504                 final int[] pixels;
505
506                 public DataImage(int w, int h, int[] pixels) {
507                         this.w = w;
508                         this.h = h;
509                         this.stride = w;
510                         this.pixels = pixels;
511                         this.source = new MemoryImageSource(w, h, pixels, 0, w);
512                         this.source.setAnimated(true);
513                         this.source.setFullBufferUpdates(true);
514                         this.image = Toolkit.getDefaultToolkit().createImage(source);
515                 }
516
517                 @Override
518                 protected void paintComponent(Graphics g) {
519                         super.paintComponent(g);
520                         g.drawImage(image, 0, 0, this);
521                 }
522         }
523
524         void swing_show(int w, int h, int[] pixels) {
525                 JFrame window;
526                 DataImage image = new DataImage(w, h, pixels);
527
528                 window = new JFrame("mandelbrot");
529                 window.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
530                 window.setContentPane(image);
531                 window.setSize(w, h);
532                 window.setVisible(true);
533         }
534
535         IntArray loadSPIRV0(String name) throws IOException {
536                 // hmm any way to just load this directly?
537                 try (InputStream is = TestMandelbrot.class.getResourceAsStream(name)) {
538                         ByteBuffer bb = ByteBuffer.allocateDirect(8192).order(ByteOrder.nativeOrder());
539                         int length = Channels.newChannel(is).read(bb);
540
541                         bb.position(0);
542                         bb.limit(length);
543
544                         return IntArray.create(MemorySegment.ofByteBuffer(bb));
545                 }
546         }
547
548         IntArray loadSPIRV(String name) throws IOException {
549                 try (InputStream is = TestMandelbrot.class.getResourceAsStream(name)) {
550                         MemorySegment seg = ((SegmentAllocator)scope).allocateArray(Memory.INT, 2048);
551                         int length = Channels.newChannel(is).read(seg.asByteBuffer());
552
553                         return IntArray.create(seg.asSlice(0, length));
554                 }
555         }
556
557         /**
558          * This finds the memory type index for the memory on a specific device.
559          */
560         static int find_memory_type(VkPhysicalDeviceMemoryProperties memory, int typeMask, int query) {
561                 VkMemoryType mtypes = memory.getMemoryTypes();
562
563                 for (int i = 0; i < memory.getMemoryTypeCount(); i++) {
564                         if (((1 << i) & typeMask) != 0 && ((mtypes.getPropertyFlagsAtIndex(i) & query) == query))
565                                 return i;
566                 }
567                 return -1;
568         }
569
570         public static int VK_MAKE_API_VERSION(int variant, int major, int minor, int patch) {
571                 return (variant << 29) | (major << 22) | (minor << 12) | patch;
572         }
573
574         void demo() throws Exception {
575                 mandelbrot_cs = loadSPIRV("mandelbrot.bin");
576
577                 init_instance();
578                 init_debug();
579
580                 init_device();
581
582                 dst = init_buffer(dstBufferSize,
583                         VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
584                         VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
585
586                 init_descriptor();
587
588                 init_pipeline();
589                 init_command_buffer();
590
591                 System.out.printf("Calculating %dx%d\n", WIDTH, HEIGHT);
592                 execute();
593                 //System.out.println("Saving ...");
594                 //save_result();
595                 System.out.println("Showing ...");
596                 show_result();
597                 System.out.println("Done.");
598
599                 shutdown();
600         }
601
602
603         public static void main(String[] args) throws Throwable {
604                 System.loadLibrary("vulkan");
605
606                 new TestMandelbrot().demo();
607         }
608 }