beamformer_core.c (81334B)
1 /* See LICENSE for license details. */ 2 /* TODO(rnp): 3 * [ ]: backtrace dumping on SIGSEGV 4 * [ ]: cooperative shared memory loading in decode shader 5 * [ ]: refactor: save filter parameters with rest of parameters, whole slot thing is dumb 6 * [ ]: upload previously exported data for display. maybe this is a UI thing but doing it 7 * programatically would be nice. 8 * [ ]: Add interface for multi frame upload. RF upload already uses an offset into SM so 9 * that part works fine. We just need a way of specify a multi frame upload. (Data must 10 * be organized for simple offset access per frame). 11 * [ ]: refactor: do_compute should build its own "command graph" which tracks 12 * dependencies better. It is very important that unnecessary barriers are 13 * not placed between compute stages which requires knowledge of the entire 14 * graph. 15 * [ ]: refactor: replace UploadRF with just the scratch_rf_size variable, 16 * use below to spin wait in library 17 * [ ]: utilize umonitor/umwait (intel), monitorx/mwaitx (amd), and wfe/sev (aarch64) 18 * for power efficient low latency waiting 19 * [ ]: BeamformWorkQueue -> BeamformerWorkQueue 20 * [ ]: refactor: work queue needs a cleanup, we should only have a single one 21 * - that queue isn't really considered hot so a lock is probably fine 22 * [ ]: bug: reinit cuda on hot-reload 23 * 24 * [ ]: export for special frames/data 25 * - Color Map Data 26 * - Recursive Imaging Result 27 * - Incoherent sum 28 * 29 * [ ]: Tiled Array Handling 30 * [ ]: add tile count uv2 31 * [ ]: make xdc_transform an array 32 * [ ]: modify CPU DAS dispatch code to lookup xdc_transform based on tile index 33 * [ ]: modify CPU DAS dispatch code to set rf_element_offset based on tile index 34 * - need to check if this works for HERCULES or if the shader just needs to 35 * know about the extra tiles 36 * [ ]: write simple backpropagation code to optimize 2D tile position and 2D tilt 37 * (4 variables per tile). 38 * - two tests: 39 * 1. Maximize value of wire target 40 * 2. Maximize value of single cyst contrast 41 * 42 * [ ]: Recursive Imaging Handling 43 * [ ]: add array of image offsets to Compute Array Parameters 44 * [ ]: add array of weighting coeffiecents to Compute Array Parameters 45 * [ ]: Update 2D sum shader to use these 46 * 47 * [ ]: Power Doppler 48 * [ ]: also needs array of image offsets 49 * [ ]: make modified filter that grabs each sample from a different image offset 50 * - NOTE: this can't use existing striding mechanism because one image may 51 * be at the start of the ring buffer and another at the end of the 52 * ring buffer and the image size may not cleanly divide the ring buffer size. 53 * - Probably want this shader to just output the estimated power map directly. 54 * [ ]: make a modified render shader that takes two images: one structural and one that 55 * is used to index into a color map. (or second render pass that applies color overlay) 56 */ 57 58 #include "base_platform.h" 59 60 #if defined(BEAMFORMER_DEBUG) && !defined(BEAMFORMER_EXPORT) && OS_WINDOWS 61 #define BEAMFORMER_EXPORT __declspec(dllexport) 62 #endif 63 64 #include "beamformer_internal.h" 65 66 #define GPU_RESOURCE_HASH_TABLE_COUNT 256 67 68 typedef struct GPUResource GPUResource; 69 struct GPUResource { 70 str8 name; 71 u64 size; 72 u64 offset; 73 u64 alignment; 74 75 void *data; 76 77 u64 hash; 78 79 GPUResource *next; 80 GPUResource *hash_next, *hash_prev; 81 }; 82 typedef struct {GPUResource *first, *last;} GPUResourceHashBucket; 83 84 typedef struct { 85 Arena *arena; 86 u64 position; 87 88 GPUResource *resource_list; 89 GPUResourceHashBucket hash_table[GPU_RESOURCE_HASH_TABLE_COUNT]; 90 } GPUResourceBuilder; 91 92 read_only global BeamformerFrame beamformer_nil_frame; 93 read_only global BeamformerComputePlan beamformer_nil_compute_plan; 94 95 global BeamformerCtx *beamformer_context; 96 global BeamformerInput *beamformer_input; 97 global f32 dt_for_frame; 98 99 #define beamformer_frame_arena() (beamformer_context->frame_arenas[beamformer_context->frame_index % countof(beamformer_context->frame_arenas)]) 100 #define beamformer_registers() (&beamformer_context->registers->v) 101 #define beamformer_push_registers(...) beamformer_push_registers_(&(BeamformerRegisters){beamformer_registers_init_literal __VA_ARGS__}) 102 #define BeamformerRegistersScope(...) DeferLoop(beamformer_push_registers(__VA_ARGS__), beamformer_pop_registers()) 103 #define beamformer_command(name, ...) beamformer_push_command(name, &(BeamformerRegisters){beamformer_registers_init_literal __VA_ARGS__}) 104 105 function BeamformerRegisters * 106 beamformer_pop_registers(void) 107 { 108 BeamformerRegisters *result = &beamformer_context->registers->v; 109 SLLStackPop(beamformer_context->registers, next); 110 if (beamformer_context->registers == 0) 111 beamformer_context->registers = &beamformer_context->base_registers; 112 return result; 113 } 114 115 function BeamformerRegisters * 116 beamformer_push_registers_(BeamformerRegisters *registers) 117 { 118 BeamformerRegistersNode *node = push_struct(beamformer_frame_arena(), BeamformerRegistersNode); 119 BeamformerRegisters *result = &node->v; 120 memory_copy(result, registers, sizeof(node->v)); 121 SLLStackPush(beamformer_context->registers, node, next); 122 return result; 123 } 124 125 function void 126 beamformer_command_list_push_new(Arena *arena, BeamformerCommandList *commands, str8 name, BeamformerRegisters *registers) 127 { 128 BeamformerCommandNode *node = push_struct(arena, BeamformerCommandNode); 129 node->command.registers = push_struct_no_zero(arena, BeamformerRegisters); 130 node->command.name = push_str8(arena, name); 131 memory_copy(node->command.registers, registers, sizeof(*registers)); 132 DLLInsertLast(0, commands->first, commands->last, node, next, prev); 133 commands->count += 1; 134 } 135 136 function void 137 beamformer_push_command(str8 name, BeamformerRegisters *registers) 138 { 139 beamformer_command_list_push_new(beamformer_frame_arena(), beamformer_context->command_queues + 0, 140 name, registers); 141 } 142 143 function BeamformerCommandKind 144 beamformer_command_kind_from_string(str8 s) 145 { 146 BeamformerCommandKind result = BeamformerCommandKind_Nil; 147 for EachElement(beamformer_command_infos, it) { 148 if (str8_equal(beamformer_command_infos[it].string, s)) { 149 result = (BeamformerCommandKind)it; 150 break; 151 } 152 } 153 return result; 154 } 155 156 function BeamformerPanelKind 157 beamformer_panel_kind_from_string(str8 s) 158 { 159 BeamformerPanelKind result = BeamformerPanelKind_Nil; 160 for EachElement(beamformer_panel_infos, it) { 161 if (str8_equal(beamformer_panel_infos[it].string, s)) { 162 result = (BeamformerPanelKind)it; 163 break; 164 } 165 } 166 return result; 167 } 168 169 function BeamformerFrame * 170 beamformer_frame_from_index(u64 index) 171 { 172 BeamformerFrame *result = &beamformer_nil_frame; 173 if (index < countof(beamformer_context->compute_context.backlog.frames)) { 174 BeamformerFrame *frame = beamformer_context->compute_context.backlog.frames + index; 175 if (frame->timeline_valid_value != 0) 176 result = frame; 177 } 178 return result; 179 } 180 181 function b32 182 beamformer_frame_valid(u64 index) 183 { 184 b32 result = beamformer_frame_from_index(index) != &beamformer_nil_frame; 185 return result; 186 } 187 188 function void 189 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block) 190 { 191 assert(block < countof(cc->compute_plans)); 192 BeamformerComputePlan *cp = cc->compute_plans[block]; 193 if (cp) { 194 gpu_buffer_release(&cp->gpu_temp_arena); 195 cc->compute_plans[block] = 0; 196 SLLPushFreelist(cp, cc->compute_plan_freelist); 197 } 198 } 199 200 function GPUResource * 201 gpu_resource_from_hash(GPUResourceBuilder *rb, u64 hash) 202 { 203 GPUResource *result = 0; 204 205 GPUResourceHashBucket *hb = rb->hash_table + (hash % GPU_RESOURCE_HASH_TABLE_COUNT); 206 for (GPUResource *r = hb->first; r; r = r->hash_next) { 207 if (hash == r->hash) { 208 result = r; 209 break; 210 } 211 } 212 213 return result; 214 } 215 216 typedef struct { 217 str8 name; 218 u64 align; 219 u64 size; 220 void *data; 221 } GPUResourcePushInfo; 222 #define gpu_resource_push(rb, t, count, ...) gpu_resource_push_(rb, (GPUResourcePushInfo){\ 223 .align = Max(alignof(t), 16), \ 224 .size = sizeof(t) * count, \ 225 __VA_ARGS__}) 226 227 function u32 228 gpu_resource_push_(GPUResourceBuilder *rb, GPUResourcePushInfo info) 229 { 230 assert(info.size > 0 && info.name.length > 0 && IsPowerOfTwo(info.align)); 231 232 u64 hash = u64_hash_from_str8(info.name); 233 GPUResource *r = gpu_resource_from_hash(rb, hash); 234 if (!r) { 235 r = push_struct(rb->arena, GPUResource); 236 GPUResourceHashBucket *hb = rb->hash_table + (hash % GPU_RESOURCE_HASH_TABLE_COUNT); 237 DLLInsert(0, hb->first, hb->last, r, hash_next, hash_prev); 238 SLLStackPush(rb->resource_list, r, next); 239 240 r->hash = hash; 241 r->name = info.name; 242 r->alignment = Max(16, info.align); 243 r->offset = AlignUpPowerOfTwo(rb->position, r->alignment); 244 r->size = info.size; 245 rb->position = r->offset + r->size; 246 } 247 248 u32 result = r->offset; 249 250 // NOTE(rnp): if this is a new resource and no data is provided it is likely 251 // a temporary GPU side buffer. if this is not a new resource and no data 252 // is provided then maybe it is shared and someone else will provide it 253 if (info.data) r->data = info.data; 254 255 return result; 256 } 257 258 function GPUResourceBuilder * 259 gpu_resource_build_begin(Arena *arena) 260 { 261 GPUResourceBuilder *result = push_struct(arena, GPUResourceBuilder); 262 result->arena = arena; 263 return result; 264 } 265 266 function void 267 gpu_resource_build_end(GPUResourceBuilder *rb, GPUBuffer *buffer) 268 { 269 u64 size = gpu_round_up_to_sync_size(rb->position, 64); 270 if (size != (u64)buffer->size) { 271 gpu_buffer_allocate(buffer, (GPUBufferAllocateInfo){ 272 .size = size, 273 .flags = GPUUsageFlag_HostWrite, 274 .label = push_str8_f(rb->arena, "GPU Temp Arena [%p]", buffer), 275 }); 276 } 277 278 ////////////////////////////////////// 279 // NOTE(rnp): upload data 280 u64 last_wait_value = 0; 281 for (GPUResource *r = rb->resource_list; r; r = r->next) 282 if (r->data) 283 last_wait_value = gpu_buffer_range_upload(buffer, r->data, r->offset, r->size, 0); 284 285 // TODO(rnp): cleanup this pointless stall 286 if (vk_buffer_needs_sync(buffer)) 287 gpu_host_wait_timeline(GPUTimeline_Transfer, last_wait_value, -1ULL); 288 } 289 290 function BeamformerComputePlan * 291 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena) 292 { 293 assert(block < countof(cc->compute_plans)); 294 BeamformerComputePlan *result = cc->compute_plans[block]; 295 if (!result) { 296 result = SLLPopFreelist(cc->compute_plan_freelist); 297 if (!result) result = push_struct_no_zero(arena, BeamformerComputePlan); 298 zero_struct(result); 299 cc->compute_plans[block] = result; 300 301 result->ui_voxel_transform = m4_identity(); 302 } 303 return result; 304 } 305 306 function BeamformerFilter * 307 beamformer_filter_create(Arena *arena, BeamformerFilterParameters fp) 308 { 309 BeamformerFilter *result = push_struct(arena, BeamformerFilter); 310 switch (fp.kind) { 311 case BeamformerFilterKind_Kaiser:{ 312 /* TODO(rnp): this should also support complex */ 313 /* TODO(rnp): implement this as an IFIR filter instead to reduce computation */ 314 result->data = kaiser_low_pass_filter(arena, fp.kaiser.cutoff_frequency, fp.sampling_frequency, 315 fp.kaiser.beta, (i32)fp.kaiser.length); 316 result->length = (i32)fp.kaiser.length; 317 result->time_delay = (f32)result->length / 2.0f / fp.sampling_frequency; 318 }break; 319 320 case BeamformerFilterKind_MatchedChirp:{ 321 typeof(fp.matched_chirp) *mc = &fp.matched_chirp; 322 f32 fs = fp.sampling_frequency; 323 result->length = (i32)(mc->duration * fs); 324 if (fp.complex) { 325 result->data = baseband_chirp(arena, mc->min_frequency, mc->max_frequency, fs, result->length, 1, 0.5f); 326 result->time_delay = complex_filter_first_moment(result->data, result->length, fs); 327 } else { 328 result->data = rf_chirp(arena, mc->min_frequency, mc->max_frequency, fs, result->length, 1); 329 result->time_delay = real_filter_first_moment(result->data, result->length, fs); 330 } 331 }break; 332 333 InvalidDefaultCase; 334 } 335 336 result->parameters = fp; 337 return result; 338 } 339 340 function iv3 341 das_valid_points(iv3 points) 342 { 343 iv3 result; 344 result.x = Max(points.x, 1); 345 result.y = Max(points.y, 1); 346 result.z = Max(points.z, 1); 347 return result; 348 } 349 350 function GPUBuffer * 351 beamformer_gpu_buffer_from_frame(BeamformerFrame *frame) 352 { 353 GPUBuffer *result = beamformer_context->compute_context.backlog.buffer; 354 if (!Between(frame->gpu_pointer, result->gpu_pointer, result->gpu_pointer + result->size)) { 355 result = 0; 356 BeamformerComputePlan *cp = beamformer_context->compute_context.compute_plans[frame->parameter_block]; 357 if (cp) { 358 result = &cp->gpu_temp_arena; 359 assert(Between(frame->gpu_pointer, result->gpu_pointer, result->gpu_pointer + result->size)); 360 } 361 } 362 return result; 363 } 364 365 function u64 366 beamformer_frame_byte_size(iv3 points, BeamformerDataKind kind) 367 { 368 u64 result = points.x * points.y * points.z * beamformer_data_kind_byte_size[kind]; 369 result = round_up_to(result, 64); 370 return result; 371 } 372 373 function u64 374 beamformer_incoherent_frame_byte_size(iv3 points, BeamformerDataKind kind) 375 { 376 u64 result = beamformer_frame_byte_size(points, kind) / beamformer_data_kind_element_count[kind]; 377 return result; 378 } 379 380 function BeamformerFrame * 381 beamformer_frame_next(BeamformerComputeContext *cc, iv3 output_points, b32 complex) 382 { 383 BeamformerFrameBacklog *bl = &cc->backlog; 384 385 BeamformerDataKind kind = complex ? BeamformerDataKind_Float32Complex : BeamformerDataKind_Float32; 386 u64 frame_size = beamformer_frame_byte_size(output_points, kind); 387 388 // TODO(rnp): handle this somewhat gracefully (even it produces garbled output) 389 assert(frame_size <= (u64)bl->buffer->size); 390 391 if (bl->next_offset > (u64)bl->buffer->size - frame_size) 392 bl->next_offset = 0; 393 394 u64 id = bl->counter++; 395 396 BeamformerFrame *result = bl->frames + (id % countof(bl->frames)); 397 atomic_store_u64(&result->timeline_valid_value, -1ULL); 398 result->id = id & U32_MAX; 399 result->gpu_pointer = bl->buffer->gpu_pointer + bl->next_offset; 400 result->points = output_points; 401 result->data_kind = kind; 402 403 bl->next_offset += frame_size; 404 405 return result; 406 } 407 408 function void 409 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info) 410 { 411 u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer); 412 t->buffer[index] = info; 413 } 414 415 function uv3 416 layout_for_output(iv3 points) 417 { 418 uv3 result = {{1, 1, 1}}; 419 420 b32 has_x = points.x > 1; 421 b32 has_y = points.y > 1; 422 b32 has_z = points.z > 1; 423 424 u32 subgroup_size = gpu_info()->subgroup_size; 425 u32 grid_3d_z_size = Max(1, subgroup_size / (4 * 4)); 426 u32 grid_2d_y_size = Max(1, subgroup_size / 8); 427 428 switch (iv3_dimension(points)) { 429 case 1:{ 430 if (has_x) result.x = subgroup_size; 431 if (has_y) result.y = subgroup_size; 432 if (has_z) result.z = subgroup_size; 433 }break; 434 435 case 2:{ 436 if (has_x && has_y) {result.x = 8; result.y = grid_2d_y_size;} 437 if (has_x && has_z) {result.x = 8; result.z = grid_2d_y_size;} 438 if (has_y && has_z) {result.y = 8; result.z = grid_2d_y_size;} 439 }break; 440 441 case 3:{result = (uv3){{4, 4, grid_3d_z_size}};}break; 442 443 InvalidDefaultCase; 444 } 445 446 return result; 447 } 448 449 function uv3 450 dispatch_for_output(uv3 layout, iv3 points) 451 { 452 uv3 result; 453 result.x = (u32)ceil_f32((f32)points.x / layout.x); 454 result.y = (u32)ceil_f32((f32)points.y / layout.y); 455 result.z = (u32)ceil_f32((f32)points.z / layout.z); 456 return result; 457 } 458 459 typedef struct BeamformerComputeGraphNode BeamformerComputeGraphNode; 460 struct BeamformerComputeGraphNode { 461 // NOTE(rnp): will be BeamformerShaderKind_Count for root node 462 BeamformerShaderKind kind; 463 464 // NOTE(rnp): when any of input or output stride is assigned it is assumed that 465 // the shader requires a fixed layout for input, output, or both. When two adjacent 466 // nodes require incompatible layouts the second pass over the graph will insert 467 // Reshape shaders in between. 468 BeamformerDataKind input_data_kind; 469 iv3 input_stride; 470 471 BeamformerDataKind output_data_kind; 472 iv3 output_stride; 473 474 b32 requires_fp_input; 475 476 i32 user_pipeline_index; 477 478 BeamformerComputeGraphNode *prev; 479 BeamformerComputeGraphNode *next; 480 }; 481 482 typedef struct { 483 BeamformerComputeGraphNode *first; 484 BeamformerComputeGraphNode *last; 485 u64 count; 486 } BeamformerComputeGraph; 487 488 function b32 489 compute_plan_push_shader(BeamformerComputePlan *p, BeamformerComputeGraphNode *node, BeamformerShaderParameters *sp) 490 { 491 b32 result = 0; 492 if (p->pipeline.shader_count < countof(p->pipeline.shaders)) { 493 u32 index = p->pipeline.shader_count++; 494 p->pipeline.shaders[index] = node->kind; 495 zero_struct(p->shader_descriptors + index); 496 p->pipeline.parameters[index] = sp ? *sp : (BeamformerShaderParameters){0}; 497 498 p->shader_descriptors[index].input_data_kind = node->input_data_kind; 499 p->shader_descriptors[index].output_data_kind = node->output_data_kind; 500 501 result = 1; 502 } 503 return result; 504 } 505 506 function BeamformerComputeGraphNode * 507 push_compute_graph_node(BeamformerComputeGraph *graph, BeamformerShaderKind kind, Arena *arena) 508 { 509 BeamformerComputeGraphNode *result = push_struct(arena, BeamformerComputeGraphNode); 510 if (graph) { 511 DLLInsertLast(0, graph->first, graph->last, result, next, prev); 512 graph->count++; 513 } 514 result->kind = kind; 515 result->user_pipeline_index = -1; 516 // NOTE(rnp): initially don't care data kind 517 result->input_data_kind = BeamformerDataKind_Count; 518 result->output_data_kind = BeamformerDataKind_Count; 519 return result; 520 } 521 522 function void 523 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, Arena *scratch) 524 { 525 b32 run_hilbert = 0; 526 b32 demodulate = 0; 527 528 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 529 switch (pb->pipeline.shaders[i]) { 530 case BeamformerShaderKind_Hilbert:{run_hilbert = 1;}break; 531 case BeamformerShaderKind_Demodulate:{demodulate = 1;}break; 532 default:{}break; 533 } 534 } 535 536 if (demodulate) run_hilbert = 0; 537 538 f32 sampling_frequency = pb->parameters.sampling_frequency; 539 u32 input_sample_count = pb->parameters.sample_count; 540 u32 acquisition_count = pb->parameters.acquisition_count; 541 u32 decimation_rate = Max(pb->parameters.decimation_rate, 1); 542 543 cp->raw_channel_byte_stride = pb->parameters.sample_count * pb->parameters.acquisition_count 544 * beamformer_data_kind_byte_size[pb->pipeline.data_kind]; 545 546 BeamformerDataKind input_data_kind = pb->pipeline.data_kind; 547 if (demodulate) { 548 switch (input_data_kind) { 549 case BeamformerDataKind_Int16:{ input_data_kind = BeamformerDataKind_Int16Complex; }break; 550 case BeamformerDataKind_Float16:{input_data_kind = BeamformerDataKind_Float16Complex;}break; 551 case BeamformerDataKind_Float32:{input_data_kind = BeamformerDataKind_Float32Complex;}break; 552 default:{}break; 553 } 554 input_sample_count /= (2 * decimation_rate); 555 sampling_frequency /= (2 * decimation_rate); 556 } 557 558 cp->iq_pipeline = beamformer_data_kind_complex[input_data_kind] || run_hilbert; 559 560 BeamformerDataKind das_data_kind = cp->iq_pipeline ? BeamformerDataKind_Float32Complex 561 : BeamformerDataKind_Float32; 562 563 cp->channel_count = pb->parameters.channel_count; 564 u32 chunk_channel_count = Min(cp->channel_count, BeamformerChunkChannelCount); 565 566 cp->rf_size = input_sample_count * pb->parameters.acquisition_count * chunk_channel_count 567 * beamformer_data_kind_byte_size[das_data_kind]; 568 569 i64 buffer_size = PING_PONG_BUFFER_SLOTS * round_up_to(cp->rf_size, 64); 570 if (beamformer_context->compute_context.ping_pong_buffer.size < buffer_size) { 571 b32 cuda = cuda_supported(); 572 GPUBufferAllocateInfo allocate_info = { 573 .size = buffer_size, 574 .export = cuda ? &beamformer_context->compute_context.ping_pong_export_handle : 0, 575 .label = str8("PingPongBuffer"), 576 }; 577 gpu_buffer_allocate(&beamformer_context->compute_context.ping_pong_buffer, allocate_info); 578 579 // TODO(rnp): figure out how to share with CUDA 580 // IMPORTANT: on linux the handle is returned to os and should be cleared after import 581 // see usage of glImportMemoryFdEXT and surrounding code in ui.c for examples 582 if (cuda) { 583 } 584 } 585 586 read_only local_persist BeamformerDataKind data_kind_to_element_kind[] = { 587 [BeamformerDataKind_Int16] = BeamformerDataKind_Float16, 588 [BeamformerDataKind_Float16] = BeamformerDataKind_Float16, 589 [BeamformerDataKind_Float32] = BeamformerDataKind_Float32, 590 [BeamformerDataKind_Int16Complex] = BeamformerDataKind_Float16, 591 [BeamformerDataKind_Float16Complex] = BeamformerDataKind_Float16, 592 [BeamformerDataKind_Float32Complex] = BeamformerDataKind_Float32, 593 }; 594 595 read_only local_persist BeamformerDataKind data_kind_to_fp_kind[] = { 596 [BeamformerDataKind_Int16] = BeamformerDataKind_Float16, 597 [BeamformerDataKind_Float16] = BeamformerDataKind_Float16, 598 [BeamformerDataKind_Float32] = BeamformerDataKind_Float32, 599 [BeamformerDataKind_Int16Complex] = BeamformerDataKind_Float16Complex, 600 [BeamformerDataKind_Float16Complex] = BeamformerDataKind_Float16Complex, 601 [BeamformerDataKind_Float32Complex] = BeamformerDataKind_Float32Complex, 602 }; 603 604 ////////////////////////////////////// 605 // NOTE(rnp): First Pass: build initial graph and insert hard layout constraints 606 BeamformerComputeGraph graph = {0}; 607 BeamformerComputeGraphNode *root_node = push_compute_graph_node(&graph, BeamformerShaderKind_Count, scratch); 608 root_node->input_data_kind = input_data_kind; 609 root_node->input_stride.x = 1; // Sample Stride 610 root_node->input_stride.y = pb->parameters.sample_count * acquisition_count; // Channel Stride 611 root_node->input_stride.z = pb->parameters.sample_count; // Receive Event Stride 612 root_node->output_data_kind = input_data_kind; 613 root_node->output_stride.x = 1; // Sample Stride 614 root_node->output_stride.y = pb->parameters.sample_count * acquisition_count; // Channel Stride 615 root_node->output_stride.z = pb->parameters.sample_count; // Receive Event Stride 616 617 for EachIndex(pb->pipeline.shader_count, it) { 618 // NOTE(rnp): skip unnecessary shaders 619 switch (pb->pipeline.shaders[it]) { 620 case BeamformerShaderKind_Hilbert:{if (!run_hilbert) continue;}break; 621 622 case BeamformerShaderKind_Decode:{ 623 if (pb->parameters.decode_mode == BeamformerDecodeMode_None) 624 continue; 625 }break; 626 627 case BeamformerShaderKind_Sum: 628 case BeamformerShaderKind_MinMax: 629 { 630 // NOTE(rnp): currently unsupported 631 continue; 632 }break; 633 634 default:{}break; 635 } 636 637 BeamformerComputeGraphNode *node = push_compute_graph_node(&graph, pb->pipeline.shaders[it], scratch); 638 node->user_pipeline_index = (i32)it; 639 switch (pb->pipeline.shaders[it]) { 640 case BeamformerShaderKind_Decode:{ 641 b32 low_precision = beamformer_data_kind_element_size[input_data_kind] < 4; 642 b32 use_coop_matrix = gpu_info()->cooperative_matrix && 643 low_precision && 644 (acquisition_count % 16 == 0) && 645 (chunk_channel_count % 16 == 0); 646 647 node->requires_fp_input = 1; 648 649 // NOTE(rnp): fixed input layout required for reasonable performance 650 node->input_stride.x = chunk_channel_count * acquisition_count; 651 node->input_stride.y = acquisition_count; 652 node->input_stride.z = 1; 653 654 if (use_coop_matrix) { 655 node->input_data_kind = BeamformerDataKind_Float16; 656 node->output_data_kind = data_kind_to_element_kind[das_data_kind]; 657 node->output_stride = node->input_stride; 658 } 659 }break; 660 661 case BeamformerShaderKind_DAS:{ 662 // NOTE(rnp): if we aren't decoding we can let the previous stage's 663 // data kind pass through as long as it is floating point 664 node->requires_fp_input = 1; 665 if (pb->parameters.decode_mode != BeamformerDecodeMode_None) 666 node->input_data_kind = das_data_kind; 667 668 node->input_stride.x = 1; // Sample Stride 669 node->input_stride.y = input_sample_count * acquisition_count; // Channel Stride 670 node->input_stride.z = input_sample_count; // Receive Event Stride 671 node->output_stride.x = 1; 672 node->output_stride.y = cp->output_points.x; 673 node->output_stride.z = cp->output_points.x * cp->output_points.y; 674 node->output_data_kind = das_data_kind; 675 676 // NOTE(rnp): insert implicit CoherencyWeighting node 677 if (pb->parameters.coherency_weighting) 678 node = push_compute_graph_node(&graph, BeamformerShaderKind_CoherencyWeighting, scratch); 679 }break; 680 681 default:{}break; 682 } 683 } 684 685 ////////////////////////////////////// 686 // NOTE(rnp): Second Pass: resolve layout constraints 687 for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) { 688 b32 needs_reshape = 0; 689 690 // NOTE(rnp): data strides 691 { 692 b32 input_dont_care = bv3_any(iv3_equal(node->input_stride, (iv3){0})); 693 b32 prev_output_dont_care = bv3_any(iv3_equal(node->prev->output_stride, (iv3){0})); 694 695 if (prev_output_dont_care && !input_dont_care) 696 node->prev->output_stride = node->input_stride; 697 698 if (!prev_output_dont_care && input_dont_care) 699 node->input_stride = node->prev->output_stride; 700 701 if (prev_output_dont_care && input_dont_care) 702 node->input_stride = node->prev->output_stride = node->prev->input_stride; 703 704 needs_reshape |= !bv3_all(iv3_equal(node->input_stride, node->prev->output_stride)); 705 } 706 707 // NOTE(rnp): data kinds 708 { 709 b32 input_dont_care = node->input_data_kind == BeamformerDataKind_Count; 710 b32 prev_output_dont_care = node->prev->output_data_kind == BeamformerDataKind_Count; 711 712 if (prev_output_dont_care && !input_dont_care) 713 node->prev->output_data_kind = node->input_data_kind; 714 715 if (!prev_output_dont_care && input_dont_care) 716 node->input_data_kind = node->prev->output_data_kind; 717 718 if (prev_output_dont_care && input_dont_care) 719 node->input_data_kind = node->prev->output_data_kind = node->prev->input_data_kind; 720 721 if (node->requires_fp_input) { 722 node->input_data_kind = data_kind_to_fp_kind[node->input_data_kind]; 723 if (prev_output_dont_care) 724 node->prev->output_data_kind = node->input_data_kind; 725 } 726 727 needs_reshape |= node->input_data_kind != node->prev->output_data_kind; 728 } 729 730 // NOTE(rnp): insert reshape if needed 731 if (needs_reshape) { 732 BeamformerComputeGraphNode *new = push_compute_graph_node(0, BeamformerShaderKind_Reshape, scratch); 733 BeamformerComputeGraphNode *last = node->prev; 734 DLLInsertLast(0, node, last, new, next, prev); 735 graph.count++; 736 new->input_data_kind = new->prev->output_data_kind; 737 new->input_stride = new->prev->output_stride; 738 new->output_data_kind = new->next->input_data_kind; 739 new->output_stride = new->next->input_stride; 740 } 741 } 742 743 // NOTE(rnp): ensure last node descriptor gets proper values for output data kind 744 if (graph.last->output_data_kind == BeamformerDataKind_Count) 745 graph.last->output_data_kind = graph.last->input_data_kind; 746 747 f32 time_offset = pb->parameters.time_offset; 748 u32 subgroup_size = gpu_info()->subgroup_size; 749 750 cp->first_image_shader_index = 0; 751 cp->pipeline.shader_count = 0; 752 753 GPUResourceBuilder *resource_builder = gpu_resource_build_begin(scratch); 754 for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) { 755 assert(node->prev->output_data_kind == node->input_data_kind); 756 assert(bv3_all(iv3_equal(node->prev->output_stride, node->input_stride))); 757 758 BeamformerShaderParameters *sp = 0; 759 if (node->user_pipeline_index >= 0) 760 sp = pb->pipeline.parameters + node->user_pipeline_index; 761 762 if (compute_plan_push_shader(cp, node, sp)) { 763 BeamformerShaderDescriptor *sd = cp->shader_descriptors + cp->pipeline.shader_count - 1; 764 765 switch (node->kind) { 766 case BeamformerShaderKind_Decode:{ 767 BeamformerDecodeBakeParameters *db = &sd->bake.Decode; 768 769 u32 decode_sample_count = input_sample_count; 770 db->DecodeMode = pb->parameters.decode_mode; 771 db->TransmitCount = pb->parameters.acquisition_count; 772 db->ChunkChannelCount = chunk_channel_count; 773 774 // NOTE(rnp): ignored when using coop matrices 775 db->OutputSampleStride = node->output_stride.x; 776 db->OutputChannelStride = node->output_stride.y; 777 db->OutputTransmitStride = node->output_stride.z; 778 779 db->ToProcess = 1; 780 781 b32 use_coop_matrix = gpu_info()->cooperative_matrix && 782 node->input_data_kind == BeamformerDataKind_Float16 && 783 (db->TransmitCount % 16 == 0) && 784 (chunk_channel_count % 16 == 0); 785 if (use_coop_matrix) { 786 // TODO(rnp): shared memory for larger sizes 787 sd->layout = (uv3){{subgroup_size, 1, 1}}; 788 789 if (demodulate) 790 decode_sample_count *= 2; 791 792 sd->compile_flags |= BeamformerDecodeCompileFlags_CooperativeMatrix; 793 db->CooperativeMatrixM = 16; 794 db->CooperativeMatrixN = 16; 795 db->CooperativeMatrixK = 16; 796 797 sd->dispatch.x = db->TransmitCount / db->CooperativeMatrixN; 798 sd->dispatch.y = chunk_channel_count / db->CooperativeMatrixM; 799 sd->dispatch.z = decode_sample_count; 800 } else if (db->TransmitCount > 40) { 801 sd->compile_flags |= BeamformerDecodeCompileFlags_UseSharedMemory; 802 803 if (db->TransmitCount == 48) 804 db->ToProcess = db->TransmitCount / 16; 805 806 b32 use_16x = db->TransmitCount == 48 || db->TransmitCount == 80 || 807 db->TransmitCount == 96 || db->TransmitCount == 160; 808 sd->layout.x = use_16x ? 16 : 32; 809 sd->layout.y = 4; 810 sd->layout.z = 1; 811 812 sd->dispatch.x = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.x / (f32)db->ToProcess); 813 sd->dispatch.y = (u32)ceil_f32((f32)chunk_channel_count / (f32)sd->layout.y); 814 sd->dispatch.z = (u32)ceil_f32((f32)decode_sample_count / (f32)sd->layout.z); 815 } else { 816 /* NOTE(rnp): register caching. using more threads will cause the compiler to do 817 * contortions to avoid spilling registers. using less gives higher performance */ 818 sd->layout = (uv3){{subgroup_size / 2, 1, 1}}; 819 820 sd->dispatch.x = (u32)ceil_f32((f32)decode_sample_count / (f32)sd->layout.x); 821 sd->dispatch.y = (u32)ceil_f32((f32)chunk_channel_count / (f32)sd->layout.y); 822 sd->dispatch.z = 1; 823 } 824 825 sd->uses_heap = 1; 826 u32 order = pb->parameters.acquisition_count; 827 db->Hadamard = gpu_resource_push(resource_builder, f16, order * order, 828 .data = make_hadamard_transpose(scratch, order, use_coop_matrix), 829 .name = str8("hadamard")); 830 }break; 831 832 case BeamformerShaderKind_Demodulate: 833 case BeamformerShaderKind_Filter: 834 { 835 b32 demod = node->kind == BeamformerShaderKind_Demodulate; 836 BeamformerFilter *f = beamformer_filter_create(scratch, cp->filter_parameters[sp->filter_slot]); 837 838 sd->compile_flags |= BeamformerFilterCompileFlags_Demodulate * demod; 839 sd->compile_flags |= BeamformerFilterCompileFlags_ComplexFilter * f->parameters.complex; 840 841 time_offset += f->time_delay; 842 843 BeamformerFilterBakeParameters *fb = &sd->bake.Filter; 844 845 sd->uses_heap = 1; 846 fb->FilterLength = (u32)f->length; 847 fb->FilterCoefficients = gpu_resource_push(resource_builder, f32, f->length * (f->parameters.complex ? 2 : 1), 848 .data = f->data, 849 .name = push_str8_f(scratch, "filter_%u", sp->filter_slot)); 850 851 fb->SampleCount = input_sample_count; 852 fb->DecimationRate = demod ? decimation_rate : 1; 853 854 b32 deinterleave = beamformer_data_kind_complex[node->input_data_kind] && 855 !beamformer_data_kind_complex[node->output_data_kind]; 856 if (deinterleave) 857 fb->BatchSampleCount = chunk_channel_count * input_sample_count * pb->parameters.acquisition_count; 858 859 fb->OutputSampleStride = node->output_stride.x; 860 fb->OutputChannelStride = node->output_stride.y; 861 fb->OutputTransmitStride = node->output_stride.z; 862 863 fb->InputSampleStride = node->input_stride.x; 864 fb->InputChannelStride = node->input_stride.y; 865 fb->InputTransmitStride = node->input_stride.z; 866 867 /* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating 868 * between sampling the I portion and the Q portion of an IQ signal. Therefore there 869 * is an implicit decimation factor of 2 which must always be included. All code here 870 * assumes that the signal was sampled in such a way that supports this operation. 871 * To recover IQ[n] from the sampled data (RF[n]) we do the following: 872 * I[n] = RF[n] 873 * Q[n] = RF[n + 1] 874 * IQ[n] = I[n] - j*Q[n] 875 */ 876 if (demod) { 877 fb->DemodulationFrequency = pb->parameters.demodulation_frequency; 878 fb->SamplingFrequency = pb->parameters.sampling_frequency / 2; 879 } 880 881 sd->layout = (uv3){{subgroup_size, 1, 1}}; 882 sd->dispatch.x = (u32)ceil_f32((f32)input_sample_count / (f32)sd->layout.x); 883 sd->dispatch.y = (u32)ceil_f32((f32)chunk_channel_count / (f32)sd->layout.y); 884 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z); 885 }break; 886 887 case BeamformerShaderKind_DAS:{ 888 cp->first_image_shader_index = cp->pipeline.shader_count; 889 890 BeamformerDASBakeParameters *db = &sd->bake.DAS; 891 db->SamplingFrequency = sampling_frequency; 892 db->DemodulationFrequency = pb->parameters.demodulation_frequency; 893 db->SpeedOfSound = pb->parameters.speed_of_sound; 894 db->TimeOffset = time_offset; 895 db->FNumber = pb->parameters.f_number; 896 db->AcquisitionKind = pb->parameters.acquisition_kind; 897 db->SampleCount = input_sample_count; 898 db->ReceiveChannelCount = pb->parameters.channel_count; 899 db->AcquisitionCount = pb->parameters.acquisition_count; 900 db->ChunkChannelCount = chunk_channel_count; 901 db->InterpolationMode = pb->parameters.interpolation_mode; 902 db->TransmitAngle = pb->parameters.focal_vector.E[0]; 903 db->FocusDepth = pb->parameters.focal_vector.E[1]; 904 db->ReadiGroupCount = pb->parameters.readi_group_count; 905 db->OutputSizeX = cp->output_points.x; 906 db->OutputSizeY = cp->output_points.y; 907 db->OutputSizeZ = cp->output_points.z; 908 db->TransmitReceiveOrientation = pb->parameters.transmit_receive_orientation; 909 910 // NOTE(rnp): old gcc will miscompile an assignment 911 memory_copy(cp->xdc_transform.E, pb->parameters.xdc_transform.E, sizeof(cp->xdc_transform)); 912 913 cp->voxel_transform = m4_mul(cp->ui_voxel_transform, pb->parameters.das_voxel_transform); 914 cp->xdc_element_pitch = pb->parameters.xdc_element_pitch; 915 916 memory_copy(cp->das_voxel_transform.E, cp->voxel_transform.E, sizeof(cp->voxel_transform)); 917 918 u32 id = pb->parameters.acquisition_kind; 919 b32 sparse = id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_UHERCULES; 920 b32 single_focus = pb->parameters.single_focus != 0; 921 b32 single_orientation = pb->parameters.single_orientation != 0; 922 923 sd->compile_flags |= BeamformerDASCompileFlags_CoherencyWeighting * (pb->parameters.coherency_weighting != 0); 924 sd->compile_flags |= BeamformerDASCompileFlags_SingleFocus * single_focus; 925 sd->compile_flags |= BeamformerDASCompileFlags_SingleOrientation * single_orientation; 926 sd->compile_flags |= BeamformerDASCompileFlags_Sparse * sparse; 927 928 sd->layout = layout_for_output(cp->output_points); 929 sd->dispatch = dispatch_for_output(sd->layout, cp->output_points); 930 931 if (id != BeamformerAcquisitionKind_UFORCES && id != BeamformerAcquisitionKind_FORCES && !single_focus) { 932 db->FocalVectors = gpu_resource_push(resource_builder, v2, db->AcquisitionCount, 933 .data = pb->focal_vectors, 934 .name = str8("focal_vectors")); 935 sd->uses_heap = 1; 936 } 937 938 if (id != BeamformerAcquisitionKind_UFORCES && id != BeamformerAcquisitionKind_FORCES && !single_orientation) { 939 db->TransmitReceiveOrientations = gpu_resource_push(resource_builder, u8, db->AcquisitionCount, 940 .data = pb->transmit_receive_orientations, 941 .name = str8("transmit_receive_orientations")); 942 sd->uses_heap = 1; 943 } 944 945 if (sparse) { 946 db->SparseElements = gpu_resource_push(resource_builder, i16, db->AcquisitionCount, 947 .data = pb->sparse_elements, 948 .name = str8("sparse_elements")); 949 sd->uses_heap = 1; 950 } 951 952 if (pb->parameters.coherency_weighting) { 953 db->IncoherentFrame = gpu_resource_push(resource_builder, u32, 0, 954 .size = beamformer_incoherent_frame_byte_size(cp->output_points, das_data_kind), 955 .name = str8("incoherent_buffer")); 956 sd->uses_heap = 1; 957 } 958 959 cp->readi_group = pb->parameters.readi_group; 960 if (db->ReadiGroupCount > 1) { 961 u32 order = db->ReadiGroupCount; 962 db->Hadamard = gpu_resource_push(resource_builder, f16, order * order, 963 .data = make_hadamard_transpose(scratch, order, 0), 964 .name = str8("readi_hadamard")); 965 sd->uses_heap = 1; 966 } 967 }break; 968 969 case BeamformerShaderKind_CoherencyWeighting:{ 970 // NOTE(rnp): beamformed data is stored in linear order; making the layout 2D or 3D 971 // here is just slower 972 sd->layout = (uv3){{subgroup_size, 1, 1}}; 973 sd->dispatch = dispatch_for_output(sd->layout, cp->output_points); 974 975 BeamformerCoherencyWeightingBakeParameters *cw = &sd->bake.CoherencyWeighting; 976 cw->Scale = 1.f; 977 cw->OutputVoxels = cp->output_points.x * cp->output_points.y * cp->output_points.z; 978 cw->IncoherentSum = gpu_resource_push(resource_builder, u32, 0, 979 .size = beamformer_incoherent_frame_byte_size(cp->output_points, das_data_kind), 980 .name = str8("incoherent_buffer")); 981 sd->uses_heap = 1; 982 }break; 983 984 case BeamformerShaderKind_Reshape:{ 985 BeamformerReshapeBakeParameters *rb = &sd->bake.Reshape; 986 b32 deinterleave = beamformer_data_kind_complex[node->input_data_kind] && 987 !beamformer_data_kind_complex[node->output_data_kind]; 988 b32 interleave = !beamformer_data_kind_complex[node->input_data_kind] && 989 beamformer_data_kind_complex[node->output_data_kind]; 990 assert(interleave == 0 || (interleave != deinterleave)); 991 sd->compile_flags |= BeamformerReshapeCompileFlags_Deinterleave * deinterleave; 992 sd->compile_flags |= BeamformerReshapeCompileFlags_Interleave * interleave; 993 994 rb->InputStrideX = node->input_stride.x; 995 rb->InputStrideY = node->input_stride.y; 996 rb->InputStrideZ = node->input_stride.z; 997 rb->OutputStrideX = node->output_stride.x; 998 rb->OutputStrideY = node->output_stride.y; 999 rb->OutputStrideZ = node->output_stride.z; 1000 1001 // NOTE(rnp): order doesn't really matter here but it must match the dispatch layout 1002 rb->SizeX = input_sample_count; 1003 rb->SizeY = chunk_channel_count; 1004 rb->SizeZ = acquisition_count; 1005 1006 sd->layout.x = 1; 1007 sd->layout.z = Min(subgroup_size, rb->SizeZ); 1008 sd->layout.y = subgroup_size / sd->layout.z; 1009 1010 sd->dispatch.x = (u32)(ceil_f32((f32)rb->SizeX / sd->layout.x)); 1011 sd->dispatch.y = (u32)(ceil_f32((f32)rb->SizeY / sd->layout.y)); 1012 sd->dispatch.z = (u32)(ceil_f32((f32)rb->SizeZ / sd->layout.z)); 1013 }break; 1014 1015 default:{}break; 1016 1017 #if 0 1018 case BeamformerShaderKind_Sum:{ 1019 sd->bake.data_kind = BeamformerDataKind_Float32; 1020 if (cp->iq_pipeline) 1021 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 1022 1023 sd->layout = layout_for_output(cp->output_points); 1024 sd->dispatch = dispatch_for_output(sd->layout, cp->output_points); 1025 1026 commit = 1; 1027 }break; 1028 #endif 1029 1030 } 1031 } 1032 } 1033 1034 cp->pipeline.data_kind = input_data_kind; 1035 1036 if (cp->first_image_shader_index == 0) 1037 cp->first_image_shader_index = cp->pipeline.shader_count; 1038 1039 gpu_resource_build_end(resource_builder, &cp->gpu_temp_arena); 1040 } 1041 1042 function void 1043 stream_append_shader_header(Stream *s, i32 reloadable_index, u64 gpu_heap_pointer, BeamformerShaderDescriptor *sd, uv3 layout) 1044 { 1045 stream_append_str8(s, str8("#version 460 core\n\n" 1046 "#extension GL_EXT_buffer_reference : require\n" 1047 "#extension GL_EXT_shader_16bit_storage : require\n" 1048 "#extension GL_EXT_shader_explicit_arithmetic_types : require\n\n" 1049 "#define f32 float32_t\n" 1050 "#define f16 float16_t\n" 1051 "#define s32 int32_t\n" 1052 "#define u64 uint64_t\n" 1053 "#define u32 uint32_t\n" 1054 "#define s16 int16_t\n" 1055 "#define u16 uint16_t\n" 1056 "#define u8 uint8_t\n" 1057 "#define s32vec2 i32vec2\n" 1058 "#define s16vec2 i16vec2\n" 1059 "\n")); 1060 1061 i32 header_vector_length = beamformer_shader_header_vector_lengths[reloadable_index]; 1062 i32 *header_vector = beamformer_shader_header_vectors[reloadable_index]; 1063 for (i32 index = 0; index < header_vector_length; index++) 1064 stream_append_str8(s, beamformer_shader_global_header_strings[header_vector[index]]); 1065 1066 if (layout.x != 0) { 1067 stream_append_str8(s, str8("layout(local_size_x = ")); 1068 stream_append_u64(s, layout.x); 1069 stream_append_str8(s, str8(", local_size_y = ")); 1070 stream_append_u64(s, layout.y); 1071 stream_append_str8(s, str8(", local_size_z = ")); 1072 stream_append_u64(s, layout.z); 1073 stream_append_str8(s, str8(") in;\n\n")); 1074 } 1075 1076 { 1077 u32 max_length = 0; 1078 for EachElement(beamformer_data_kind_str8, it) 1079 max_length = Max(max_length, (u32)beamformer_data_kind_str8[it].length); 1080 1081 for EachElement(beamformer_data_kind_str8, it) { 1082 stream_append_str8s(s, str8("#define DataKind_"), beamformer_data_kind_str8[it]); 1083 stream_pad(s, ' ', max_length - beamformer_data_kind_str8[it].length + 1); 1084 stream_append_u64(s, it); 1085 stream_append_byte(s, '\n'); 1086 } 1087 stream_append_byte(s, '\n'); 1088 } 1089 1090 if (gpu_heap_pointer) { 1091 stream_append_str8(s, str8("#define HeapBase u64(0x")); 1092 stream_append_hex_u64(s, gpu_heap_pointer); 1093 stream_append_str8(s, str8("ul)\n")); 1094 } 1095 1096 if (sd) { 1097 BeamformerDataKind data_kinds[] = {sd->input_data_kind, sd->output_data_kind}; 1098 str8 line_prefixes[] = {str8_comp("Input"), str8_comp("Output")}; 1099 for EachElement(data_kinds, it) { 1100 if (data_kinds[it] != BeamformerDataKind_Count) { 1101 stream_append_str8s(s, str8("#define "), line_prefixes[it], str8("DataType "), 1102 beamformer_data_kind_glsl_type[data_kinds[it]], 1103 str8("\n#define "), line_prefixes[it], str8("DataKind DataKind_"), 1104 beamformer_data_kind_str8[data_kinds[it]], 1105 str8("\n#define "), line_prefixes[it], str8("DataKindByteSize ")); 1106 stream_append_u64(s, beamformer_data_kind_byte_size[data_kinds[it]]); 1107 stream_append_byte(s, '\n'); 1108 } 1109 } 1110 stream_append_byte(s, '\n'); 1111 1112 stream_append_str8(s, str8("#define CompileFlags (0x")); 1113 stream_append_hex_u64_width(s, sd->compile_flags, 8); 1114 stream_append_str8(s, str8(")\n")); 1115 1116 i32 struct_id = beamformer_base_shader_to_bake_struct_id[reloadable_index]; 1117 if (struct_id != -1) { 1118 str8 *names = meta_struct_member_names_by_id[struct_id]; 1119 MetaStructInfo *si = meta_struct_info_by_id + struct_id; 1120 MetaStructMember *sm = meta_struct_members_by_id[struct_id]; 1121 for (u32 index = 0; index < si->member_count; index++) { 1122 str8 type = meta_kind_glsl_types[sm[index].type_id]; 1123 stream_append_str8(s, str8("layout(constant_id = ")); 1124 stream_append_u64(s, index); 1125 stream_append_str8s(s, str8(") const "), type, str8(" "), names[index], str8(" = "), type, str8("(1);\n")); 1126 } 1127 } 1128 } 1129 1130 if (!renderdoc_attached()) 1131 stream_append_str8(s, str8("\n\n#line 1\n")); 1132 } 1133 1134 function void 1135 beamformer_reload_pipeline(VulkanHandle *pipeline, BeamformerShaderReloadInfo *sris, u32 count, Arena *scratch) 1136 { 1137 assume(count <= 2); 1138 str8 paths[2]; 1139 VulkanPipelineCreateInfo infos[2]; 1140 1141 if (!BakeShaders) { 1142 for (u32 i = 0; i < count; i++) 1143 paths[i] = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), sris[i].filename_or_data); 1144 } 1145 1146 u32 push_constants_size = 0; 1147 for (u32 i = 0; i < count; i++) { 1148 Stream shader_stream = arena_stream(scratch); 1149 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[sris[i].shader]; 1150 if (i == 0) push_constants_size = beamformer_shader_push_constant_sizes[reloadable_index]; 1151 else assert(push_constants_size == beamformer_shader_push_constant_sizes[reloadable_index]); 1152 1153 stream_append_shader_header(&shader_stream, reloadable_index, sris[i].gpu_heap_pointer, 1154 sris[i].shader_descriptor, sris[i].layout); 1155 1156 str8 shader_text; 1157 if (BakeShaders) { 1158 stream_append_str8(&shader_stream, sris[i].filename_or_data); 1159 shader_text = arena_stream_commit_zero(scratch, &shader_stream); 1160 } else { 1161 str8 stream_data = arena_stream_commit(scratch, &shader_stream); 1162 str8 shader_data = os_read_entire_file(scratch, (c8 *)paths[i].data); 1163 // NOTE(rnp): kinda sucky but need to make sure these are a contiguous string 1164 shader_text = push_str8_from_parts(scratch, str8(""), stream_data, shader_data); 1165 } 1166 1167 infos[i].kind = sris[i].shader_kind; 1168 infos[i].text = shader_text; 1169 infos[i].name = beamformer_shader_names[sris[i].shader]; 1170 infos[i].specialization_data = sris[i].shader_descriptor ? &sris[i].shader_descriptor->bake : 0; 1171 infos[i].specialization_struct_id = beamformer_base_shader_to_bake_struct_id[reloadable_index]; 1172 1173 //str8 line = str8("---------------\n"); 1174 //str8 nl = str8("\n"); 1175 //os_console_log(line.data, line.length); 1176 //os_console_log(infos[i].name.data, infos[i].name.length); 1177 //os_console_log(nl.data, nl.length); 1178 //os_console_log(line.data, line.length); 1179 //os_console_log(infos[i].text.data, infos[i].text.length); 1180 //os_console_log(line.data, line.length); 1181 } 1182 1183 vk_pipeline_release(*pipeline); 1184 *pipeline = vk_pipeline(infos, count, push_constants_size); 1185 } 1186 1187 function void 1188 beamformer_reload_render_pipeline(VulkanHandle *pipeline, BeamformerShaderKind shader, Arena *scratch) 1189 { 1190 i32 index = beamformer_shader_reloadable_index_by_shader[shader]; 1191 BeamformerShaderReloadInfo infos[2] = { 1192 { 1193 .shader = shader, 1194 .shader_kind = beamformer_shader_primitive_is_vertex[index] ? VulkanShaderKind_Vertex : VulkanShaderKind_Mesh, 1195 .filename_or_data = BakeShaders ? beamformer_shader_data[index][0] 1196 : beamformer_reloadable_shader_files[index][0], 1197 }, 1198 { 1199 .shader = shader, 1200 .shader_kind = VulkanShaderKind_Fragment, 1201 .filename_or_data = BakeShaders ? beamformer_shader_data[index][1] 1202 : beamformer_reloadable_shader_files[index][1], 1203 }, 1204 }; 1205 beamformer_reload_pipeline(pipeline, infos, countof(infos), scratch); 1206 } 1207 1208 function void 1209 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena *scratch) 1210 { 1211 BeamformerParameterBlock *pb; 1212 DeferLoop(pb = beamformer_parameter_block_lock(ctx->shared_memory, block, -1), 1213 beamformer_parameter_block_unlock(ctx->shared_memory, block)) 1214 for EachBit(pb->region_update_flags, region) 1215 { 1216 pb->region_update_flags &= ~(1ul << region); 1217 switch (region) { 1218 case BeamformerParameterDirtyFlag_NotifyUI:{ 1219 atomic_store_u32(&ctx->ui_dirty_parameter_blocks, 1u << block); 1220 }break; 1221 1222 case BeamformerParameterDirtyFlag_Parameters:{ 1223 cp->output_points = das_valid_points(pb->parameters.output_points.xyz); 1224 cp->average_frames = pb->parameters.output_points.E[3]; 1225 1226 plan_compute_pipeline(cp, pb, scratch); 1227 1228 b32 heap_pointer_updated = cp->last_gpu_heap_pointer != cp->gpu_temp_arena.gpu_pointer; 1229 cp->last_gpu_heap_pointer = cp->gpu_temp_arena.gpu_pointer; 1230 1231 for (u32 shader_slot = 0; shader_slot < cp->pipeline.shader_count; shader_slot++) { 1232 u128 hash = u128_hash_from_data(cp->shader_descriptors + shader_slot, sizeof(BeamformerShaderDescriptor)); 1233 if (!u128_equal(hash, cp->shader_hashes[shader_slot]) || 1234 (cp->shader_descriptors[shader_slot].uses_heap && heap_pointer_updated)) 1235 { 1236 cp->dirty_programs |= 1 << shader_slot; 1237 } 1238 cp->shader_hashes[shader_slot] = hash; 1239 } 1240 1241 cp->acquisition_count = pb->parameters.acquisition_count; 1242 cp->acquisition_kind = pb->parameters.acquisition_kind; 1243 cp->contrast_mode = pb->parameters.contrast_mode; 1244 }break; 1245 } 1246 } 1247 } 1248 1249 function void 1250 do_compute_shader(GPUCommandList cmd, BeamformerComputePlan *cp, BeamformerFrame *frame, 1251 u64 input_pointer, u64 output_pointer, u32 shader_slot, u32 channel_offset) 1252 { 1253 BeamformerComputeContext *cc = &beamformer_context->compute_context; 1254 1255 uv3 dispatch = cp->shader_descriptors[shader_slot].dispatch; 1256 1257 gpu_command_bind_pipeline(cmd, cp->vulkan_pipelines[shader_slot]); 1258 1259 switch (cp->pipeline.shaders[shader_slot]) { 1260 1261 case BeamformerShaderKind_Decode:{ 1262 BeamformerDecodePushConstants pc = { 1263 .rf_buffer = input_pointer, 1264 .output_buffer = output_pointer, 1265 }; 1266 1267 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1268 gpu_command_dispatch_compute(cmd, dispatch); 1269 1270 cc->ping_pong_input_index = !cc->ping_pong_input_index; 1271 }break; 1272 1273 case BeamformerShaderKind_Hilbert:{ 1274 //cuda_hilbert(input_index, output_index); 1275 //cc->ping_pong_input_index = !cc->ping_pong_input_index; 1276 }break; 1277 1278 case BeamformerShaderKind_Filter: 1279 case BeamformerShaderKind_Demodulate: 1280 { 1281 BeamformerFilterPushConstants pc = { 1282 .input_buffer = input_pointer, 1283 .output_buffer = output_pointer, 1284 }; 1285 1286 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1287 gpu_command_dispatch_compute(cmd, dispatch); 1288 1289 cc->ping_pong_input_index = !cc->ping_pong_input_index; 1290 }break; 1291 1292 case BeamformerShaderKind_DAS:{ 1293 BeamformerDASPushConstants pc = { 1294 .xdc_element_pitch = cp->xdc_element_pitch, 1295 .rf_data = input_pointer, 1296 .output_frame = frame->gpu_pointer, 1297 .channel_offset = channel_offset, 1298 .readi_group = cp->readi_group, 1299 }; 1300 memory_copy(pc.voxel_transform.E, cp->das_voxel_transform.E, sizeof(pc.voxel_transform)); 1301 memory_copy(pc.xdc_transform.E, cp->xdc_transform.E, sizeof(pc.xdc_transform)); 1302 1303 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1304 gpu_command_dispatch_compute(cmd, dispatch); 1305 }break; 1306 1307 case BeamformerShaderKind_CoherencyWeighting:{ 1308 BeamformerCoherencyWeightingPushConstants pc = {.coherent_sum = frame->gpu_pointer}; 1309 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1310 gpu_command_dispatch_compute(cmd, dispatch); 1311 }break; 1312 1313 case BeamformerShaderKind_Reshape:{ 1314 BeamformerDataKind input_data_kind = cp->shader_descriptors[shader_slot].input_data_kind; 1315 BeamformerReshapeBakeParameters *rb = &cp->shader_descriptors[shader_slot].bake.Reshape; 1316 BeamformerReshapePushConstants pc = { 1317 .left_input_buffer = input_pointer, 1318 .right_input_buffer = input_pointer + rb->SizeX * rb->SizeY * rb->SizeZ 1319 * beamformer_data_kind_byte_size[input_data_kind], 1320 .output_buffer = output_pointer, 1321 }; 1322 1323 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1324 gpu_command_dispatch_compute(cmd, dispatch); 1325 1326 cc->ping_pong_input_index = !cc->ping_pong_input_index; 1327 }break; 1328 1329 // NOTE(rnp): invalid stages should be filtered in planning phase 1330 InvalidDefaultCase; 1331 } 1332 1333 #if 0 1334 switch (shader) { 1335 case BeamformerShaderKind_MinMax:{ 1336 for (u32 i = 1; i < frame->image.mip_map_levels; i++) { 1337 glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 1338 glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F); 1339 glProgramUniform1i(program, MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i); 1340 1341 u32 width = (u32)frame->dim.x >> i; 1342 u32 height = (u32)frame->dim.y >> i; 1343 u32 depth = (u32)frame->dim.z >> i; 1344 glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32)); 1345 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1346 } 1347 }break; 1348 case BeamformerShaderKind_Sum:{ 1349 u32 aframe_index = ctx->averaged_frame_index % countof(ctx->averaged_frames); 1350 BeamformerFrame *aframe = ctx->averaged_frames + aframe_index; 1351 aframe->id = ctx->averaged_frame_index; 1352 atomic_store_u32(&aframe->ready_to_present, 0); 1353 /* TODO(rnp): hack we need a better way of specifying which frames to sum; 1354 * this is fine for rolling averaging but what if we want to do something else */ 1355 assert(frame >= ctx->beamform_frames); 1356 assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames)); 1357 u32 base_index = (u32)(frame - ctx->beamform_frames); 1358 u32 to_average = (u32)cp->average_frames; 1359 u32 frame_count = 0; 1360 u32 *in_textures = push_array(&arena, u32, BeamformerMaxBacklogFrames); 1361 ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average); 1362 for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi)) 1363 in_textures[frame_count++] = it->texture; 1364 1365 assert(to_average == frame_count); 1366 1367 glProgramUniform1f(program, SUM_PRESCALE_UNIFORM_LOC, 1 / (f32)frame_count); 1368 /* NOTE: zero output before summing */ 1369 glClearTexImage(aframe->texture, 0, GL_RED, GL_FLOAT, 0); 1370 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 1371 1372 glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F); 1373 for (u32 i = 0; i < in_texture_count; i++) { 1374 glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 1375 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 1376 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1377 } 1378 1379 memory_copy(aframe->voxel_transform.E, frame->voxel_transform.E, sizeof(frame->voxel_transform)); 1380 aframe->compound_count = frame->compound_count; 1381 aframe->acquisition_kind = frame->acquisition_kind; 1382 }break; 1383 } 1384 #endif 1385 } 1386 1387 function void 1388 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena) 1389 { 1390 BeamformerComputeContext * cs = &ctx->compute_context; 1391 BeamformerSharedMemory * sm = ctx->shared_memory; 1392 1393 for (BeamformWork *work = beamform_work_queue_pop(q); 1394 work; 1395 beamform_work_queue_pop_commit(q), work = beamform_work_queue_pop(q)) 1396 { 1397 switch (work->kind) { 1398 1399 case BeamformerWorkKind_ExportBuffer:{ 1400 /* TODO(rnp): better way of handling DispatchCompute barrier */ 1401 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute); 1402 beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)work->lock, (u32)-1); 1403 BeamformerExportContext *ec = &work->export_context; 1404 switch (ec->kind) { 1405 case BeamformerExportKind_BeamformedData:{ 1406 BeamformerFrameBacklog *bl = &ctx->compute_context.backlog; 1407 u32 req_count = Clamp(ec->count, 1, bl->counter); 1408 u32 frame_idx = bl->counter - req_count; 1409 u8 *sm_output = beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size); 1410 u64 exported_size = 0; 1411 for (u32 export_count = 0; export_count < req_count; export_count++, frame_idx++) { 1412 BeamformerFrame *f = bl->frames + frame_idx % countof(bl->frames); 1413 u64 frame_size = beamformer_frame_byte_size(f->points, f->data_kind); 1414 assert((frame_size & 63) == 0); 1415 // NOTE(tkh) we don't want to assume that all req_count frames are the same size, 1416 // so we either need to count the total size of all requested frames first or 1417 // just fill up as much as possible. 1418 if (exported_size + frame_size <= ec->size) { 1419 u64 offset = f->gpu_pointer - bl->buffer->gpu_pointer; 1420 gpu_host_wait_timeline(GPUTimeline_Compute, f->timeline_valid_value, -1ULL); 1421 gpu_buffer_range_download(sm_output + exported_size, bl->buffer, offset, frame_size, 1); 1422 exported_size += frame_size; 1423 } 1424 } 1425 }break; 1426 1427 case BeamformerExportKind_Stats:{ 1428 ComputeTimingTable *table = ctx->compute_timing_table; 1429 /* NOTE(rnp): do a little spin to let this finish updating */ 1430 spin_wait(table->write_index != atomic_load_u32(&table->read_index)); 1431 ComputeShaderStats *stats = ctx->compute_shader_stats; 1432 if (sizeof(stats->table) <= ec->size) 1433 memory_copy(beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size), 1434 &stats->table, sizeof(stats->table)); 1435 }break; 1436 InvalidDefaultCase; 1437 } 1438 beamformer_shared_memory_release_lock(ctx->shared_memory, work->lock); 1439 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync); 1440 }break; 1441 1442 case BeamformerWorkKind_CreateFilter:{ 1443 BeamformerCreateFilterContext *fctx = &work->create_filter_context; 1444 u32 block = fctx->parameter_block; 1445 u32 slot = fctx->filter_slot; 1446 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena); 1447 cp->filter_parameters[slot] = fctx->parameters; 1448 }break; 1449 1450 case BeamformerWorkKind_ComputeIndirect: 1451 case BeamformerWorkKind_Compute: 1452 { 1453 push_compute_timing_info(ctx->compute_timing_table, 1454 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin}); 1455 1456 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_block, arena); 1457 if unlikely(beamformer_parameter_block_dirty(sm, work->compute_context.parameter_block)) { 1458 u32 block = work->compute_context.parameter_block; 1459 Temp scratch = temp_begin(arena); 1460 beamformer_commit_parameter_block(ctx, cp, block, arena); 1461 temp_end(scratch); 1462 } 1463 1464 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute); 1465 1466 u32 dirty_programs = atomic_swap_u32(&cp->dirty_programs, 0); 1467 static_assert(BeamformerMaxComputeShaderStages <= 32, ""); 1468 if unlikely(dirty_programs) { 1469 for EachBit(dirty_programs, slot) { 1470 assert(slot < BeamformerMaxComputeShaderStages); 1471 Temp scratch; 1472 DeferLoop(scratch = temp_begin(arena), temp_end(scratch)) 1473 { 1474 u64 gpu_heap_pointer = cp->gpu_temp_arena.gpu_pointer; 1475 i32 index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1476 BeamformerShaderReloadInfo info = { 1477 .shader = cp->pipeline.shaders[slot], 1478 .shader_kind = VulkanShaderKind_Compute, 1479 .shader_descriptor = cp->shader_descriptors + slot, 1480 .gpu_heap_pointer = cp->shader_descriptors[slot].uses_heap ? gpu_heap_pointer : 0, 1481 .filename_or_data = BakeShaders ? beamformer_shader_data[index][0] 1482 : beamformer_reloadable_shader_files[index][0], 1483 .layout = cp->shader_descriptors[slot].layout, 1484 }; 1485 beamformer_reload_pipeline(cp->vulkan_pipelines + slot, &info, 1, arena); 1486 } 1487 } 1488 } 1489 1490 atomic_store_u32(&cs->processing_compute, 1); 1491 1492 start_renderdoc_capture(); 1493 1494 i32 das_index = -1; 1495 i32 coherency_weighting = -1; 1496 for (u32 i = 0; i < cp->pipeline.shader_count; i++) { 1497 if (cp->pipeline.shaders[i] == BeamformerShaderKind_CoherencyWeighting) 1498 coherency_weighting = (i32)i; 1499 1500 if (cp->pipeline.shaders[i] == BeamformerShaderKind_DAS) 1501 das_index = (i32)i; 1502 } 1503 1504 BeamformerFrame *frame = beamformer_frame_next(cs, cp->output_points, cp->iq_pipeline); 1505 frame->acquisition_kind = cp->acquisition_kind; 1506 frame->contrast_mode = cp->contrast_mode; 1507 frame->compound_count = cp->acquisition_count; 1508 frame->parameter_block = work->compute_context.parameter_block; 1509 frame->view_plane_tag = work->compute_context.view_plane; 1510 memory_copy(frame->voxel_transform.E, cp->voxel_transform.E, sizeof(cp->voxel_transform)); 1511 1512 GPUCommandList cmd = gpu_command_list_begin(GPUTimeline_Compute); 1513 gpu_command_timestamp(cmd); 1514 1515 if (das_index >= 0) { 1516 GPUBuffer *backlog = cs->backlog.buffer; 1517 u64 frame_size = beamformer_frame_byte_size(frame->points, frame->data_kind); 1518 u64 offset = frame->gpu_pointer - backlog->gpu_pointer; 1519 gpu_command_clear_buffer(cmd, backlog, offset, frame_size, 0); 1520 } 1521 1522 if (coherency_weighting >= 0) { 1523 BeamformerCoherencyWeightingBakeParameters *cw = &cp->shader_descriptors[coherency_weighting].bake.CoherencyWeighting; 1524 GPUBuffer *gpu_arena = &cp->gpu_temp_arena; 1525 u64 coherent_size = beamformer_incoherent_frame_byte_size(frame->points, frame->data_kind); 1526 gpu_command_clear_buffer(cmd, gpu_arena, cw->IncoherentSum, coherent_size, 0); 1527 } 1528 1529 BeamformerRFBuffer *rf = &cs->rf_buffer; 1530 u32 compute_index = rf->compute_index; 1531 u32 slot = compute_index % countof(rf->upload_complete_values); 1532 1533 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1534 // TODO(rnp): this shouldn't be necessary, there should be a way of communicating 1535 // what the value will be so that the only the command wait is needed. 1536 spin_wait(atomic_load_u64(&rf->insertion_index) <= compute_index); 1537 1538 /* NOTE(rnp): if the GPU supports BAR there may be no need to synchronize 1539 * other than the above spin */ 1540 if (vk_buffer_needs_sync(&rf->buffer)) 1541 gpu_command_wait_timeline(cmd, GPUTimeline_Transfer, rf->upload_complete_values[slot]); 1542 } else { 1543 slot = (rf->compute_index - 1) % countof(rf->upload_complete_values); 1544 } 1545 1546 for (u32 channel_offset = 0; 1547 channel_offset < cp->channel_count; 1548 channel_offset += BeamformerChunkChannelCount) 1549 { 1550 u64 rf_pointer = rf->buffer.gpu_pointer + slot * rf->active_rf_size; 1551 rf_pointer += cp->raw_channel_byte_stride * channel_offset; 1552 1553 // NOTE(rnp): handle the case where the channel count is not a multiple of 1554 // BeamformerChunkChannelCount. We do not want to have to modify every shader 1555 // that might run to care about this so here we run a copy/clear to pad the 1556 // data for the last iteration. 1557 b32 special_handling = (cp->channel_count - channel_offset) < BeamformerChunkChannelCount; 1558 if unlikely(special_handling) { 1559 u32 channel_pad_count = BeamformerChunkChannelCount - (cp->channel_count - channel_offset); 1560 u32 input_index = cs->ping_pong_input_index; 1561 u64 pp_size = cs->ping_pong_buffer.size / PING_PONG_BUFFER_SLOTS; 1562 u64 copy_size = cp->raw_channel_byte_stride * (cp->channel_count - channel_offset); 1563 u64 copy_offset = rf_pointer - rf->buffer.gpu_pointer; 1564 u64 clear_size = cp->raw_channel_byte_stride * channel_pad_count; 1565 1566 gpu_command_copy_buffer(cmd, &cs->ping_pong_buffer, input_index * pp_size, &rf->buffer, copy_offset, copy_size); 1567 gpu_command_clear_buffer(cmd, &cs->ping_pong_buffer, input_index * pp_size + copy_size, clear_size, 0); 1568 gpu_command_pipeline_barrier(cmd, 1); 1569 } 1570 1571 for (u32 i = 0; i < cp->first_image_shader_index; i++) { 1572 u32 output_index = !cs->ping_pong_input_index; 1573 u32 input_index = cs->ping_pong_input_index; 1574 1575 u64 pp_size = cs->ping_pong_buffer.size / PING_PONG_BUFFER_SLOTS; 1576 u64 pp_input_pointer = cs->ping_pong_buffer.gpu_pointer + input_index * pp_size; 1577 u64 pp_output_pointer = cs->ping_pong_buffer.gpu_pointer + output_index * pp_size; 1578 1579 u64 output_pointer = pp_output_pointer; 1580 u64 input_pointer = (i == 0 && !special_handling) ? rf_pointer : pp_input_pointer; 1581 1582 if (i != 0) gpu_command_pipeline_barrier(cmd, 0); 1583 do_compute_shader(cmd, cp, frame, input_pointer, output_pointer, i, channel_offset); 1584 gpu_command_timestamp(cmd); 1585 } 1586 } 1587 1588 for (u32 i = cp->first_image_shader_index; i < cp->pipeline.shader_count; i++) { 1589 gpu_command_pipeline_barrier(cmd, 0); 1590 do_compute_shader(cmd, cp, frame, 0, 0, i, 0); 1591 gpu_command_timestamp(cmd); 1592 } 1593 u64 end_timeline_value = gpu_command_list_end(cmd, (VulkanHandle){0}, (VulkanHandle){0}); 1594 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1595 atomic_store_u64(rf->compute_complete_values + slot, end_timeline_value); 1596 atomic_add_u64(&rf->compute_index, 1); 1597 } 1598 1599 atomic_store_u64(&frame->timeline_valid_value, end_timeline_value); 1600 1601 Temp scratch; 1602 DeferLoop(scratch = temp_begin(arena), temp_end(scratch)) 1603 { 1604 /* NOTE(rnp): this blocks until work completes */ 1605 u64 count = 0; 1606 u64 *timestamps = gpu_read_timestamps(GPUTimeline_Compute, &count, arena); 1607 1608 i32 steps = ((i32)cp->channel_count / BeamformerChunkChannelCount) - 1; 1609 i32 step = 0; 1610 u32 shader_index = 0; 1611 u64 last_time = count > 0 ? timestamps[0] : 0; 1612 1613 for (u64 i = 1; i < count; i++) { 1614 push_compute_timing_info(ctx->compute_timing_table, (ComputeTimingInfo){ 1615 .kind = ComputeTimingInfoKind_Shader, 1616 .shader = cp->pipeline.shaders[shader_index], 1617 .shader_slot = shader_index, 1618 .timer_count = timestamps[i] - last_time, 1619 }); 1620 last_time = timestamps[i]; 1621 1622 shader_index++; 1623 if (shader_index == cp->first_image_shader_index && step < steps) { 1624 shader_index = 0; 1625 step++; 1626 } 1627 } 1628 } 1629 1630 cs->processing_progress = 1; 1631 1632 //if (has_sum) { 1633 if (0) { 1634 #if 0 1635 u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames)); 1636 ctx->averaged_frames[aframe_index].view_plane_tag = frame->view_plane_tag; 1637 ctx->averaged_frames[aframe_index].ready_to_present = 1; 1638 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index)); 1639 #endif 1640 } else { 1641 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame); 1642 } 1643 1644 atomic_store_u32(&cs->processing_compute, 0); 1645 1646 push_compute_timing_info(ctx->compute_timing_table, 1647 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd}); 1648 1649 end_renderdoc_capture(); 1650 }break; 1651 InvalidDefaultCase; 1652 } 1653 } 1654 } 1655 1656 function void 1657 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats) 1658 { 1659 /* TODO(rnp): we do not currently do anything to handle the potential for a half written 1660 * info item. this could result in garbage entries but they shouldn't really matter */ 1661 1662 u32 target = atomic_load_u32(&t->write_index); 1663 u32 stats_index = stats->latest_frame_index; 1664 1665 b32 has_rf = 0; 1666 f32 gpu_clocks_to_nano = 1.0e-9f * gpu_info()->timestamp_period_ns; 1667 1668 // NOTE(rnp): not equal (the index may wrap) 1669 while (t->read_index != target) { 1670 ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)]; 1671 switch (info.kind) { 1672 1673 case ComputeTimingInfoKind_ComputeFrameBegin:{ 1674 assert(t->compute_frame_active == 0); 1675 t->compute_frame_active = 1; 1676 /* NOTE(rnp): allow multiple instances of same shader to accumulate */ 1677 t->in_flight_shader_count = 0; 1678 memory_clear(t->in_flight_shader_ids, 0, sizeof(t->in_flight_shader_ids)); 1679 memory_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index])); 1680 }break; 1681 1682 case ComputeTimingInfoKind_ComputeFrameEnd:{ 1683 assert(t->compute_frame_active == 1); 1684 t->compute_frame_active = 0; 1685 stats_index = stats->latest_frame_index = (stats_index + 1) % countof(stats->table.times); 1686 stats->table.shader_count = t->in_flight_shader_count; 1687 memory_copy(stats->table.shader_ids, t->in_flight_shader_ids, sizeof(t->in_flight_shader_ids)); 1688 }break; 1689 1690 case ComputeTimingInfoKind_Shader:{ 1691 t->in_flight_shader_count = Max(t->in_flight_shader_count, info.shader_slot + 1u); 1692 t->in_flight_shader_ids[info.shader_slot] = info.shader; 1693 stats->table.times[stats_index][info.shader_slot] += info.timer_count * gpu_clocks_to_nano; 1694 }break; 1695 1696 case ComputeTimingInfoKind_RF_Data:{ 1697 stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas); 1698 f32 delta = info.timer_count / (f32)os_system_info()->timer_frequency; 1699 stats->table.rf_time_deltas[stats->latest_rf_index] = delta; 1700 has_rf = 1; 1701 }break; 1702 } 1703 /* NOTE(rnp): do this at the end so that stats table is always in a consistent state */ 1704 t->read_index++; 1705 } 1706 1707 for (u32 i = 0; i < stats->table.shader_count; i++) { 1708 f32 sum = 0; 1709 for EachElement(stats->table.times, it) 1710 sum += stats->table.times[it][i]; 1711 stats->average_times[i] = sum / countof(stats->table.times); 1712 } 1713 1714 if (has_rf) { 1715 f32 sum = 0; 1716 for EachElement(stats->table.rf_time_deltas, i) 1717 sum += stats->table.rf_time_deltas[i]; 1718 stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas); 1719 } 1720 } 1721 1722 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute) 1723 { 1724 BeamformerSharedMemory *sm = ctx->shared_memory; 1725 complete_queue(ctx, &sm->external_work_queue, arena); 1726 complete_queue(ctx, ctx->beamform_work_queue, arena); 1727 } 1728 1729 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload) 1730 { 1731 BeamformerSharedMemory *sm = ctx->shared_memory; 1732 BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace; 1733 BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF; 1734 1735 u64 rf_block_rf_size; 1736 if (atomic_load_u32(sm->locks + upload_lock) && 1737 (rf_block_rf_size = atomic_swap_u64(&sm->rf_block_rf_size, 0))) 1738 { 1739 beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)scratch_lock, (u32)-1); 1740 1741 BeamformerRFBuffer *rf = ctx->rf_buffer; 1742 1743 rf->active_rf_size = gpu_round_up_to_sync_size(rf_block_rf_size & 0xFFFFFFFFULL, 64); 1744 if unlikely(rf->buffer.size < countof(rf->upload_complete_values) * rf->active_rf_size) { 1745 GPUBufferAllocateInfo allocate_info = { 1746 .size = countof(rf->upload_complete_values) * rf->active_rf_size, 1747 .flags = GPUUsageFlag_HostWrite, 1748 .label = str8("RawRFBuffer"), 1749 }; 1750 gpu_buffer_allocate(&rf->buffer, allocate_info); 1751 } 1752 1753 u64 slot = rf->insertion_index % countof(rf->upload_complete_values); 1754 1755 /* NOTE(rnp): don't overwrite slot if the compute thread hasn't processed it */ 1756 spin_wait(atomic_load_u64(&rf->compute_index) < rf->insertion_index); 1757 gpu_host_wait_timeline(GPUTimeline_Compute, rf->compute_complete_values[slot], -1ULL); 1758 1759 assert((ctx->shared_memory_size % os_system_info()->page_size) == 0 && 1760 (os_system_info()->page_size % gpu_round_up_to_sync_size(1, 64)) == 0); 1761 u64 wait_value = gpu_buffer_range_upload(&rf->buffer, beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size), 1762 slot * rf->active_rf_size, rf->active_rf_size, 1); 1763 store_fence(); 1764 1765 beamformer_shared_memory_release_lock(ctx->shared_memory, (i32)scratch_lock); 1766 post_sync_barrier(ctx->shared_memory, upload_lock); 1767 1768 atomic_store_u64(rf->upload_complete_values + slot, wait_value); 1769 atomic_add_u64(&rf->insertion_index, 1); 1770 1771 os_wake_all_waiters(ctx->compute_worker_sync); 1772 1773 u64 current_time = os_timer_count(); 1774 push_compute_timing_info(ctx->compute_timing_table, (ComputeTimingInfo){ 1775 .kind = ComputeTimingInfoKind_RF_Data, 1776 .timer_count = current_time - rf->timestamp, 1777 }); 1778 rf->timestamp = current_time; 1779 } 1780 } 1781 1782 function void 1783 beamformer_queue_compute(BeamformerCtx *ctx, BeamformerFrame *frame, u32 parameter_block) 1784 { 1785 BeamformerSharedMemory *sm = ctx->shared_memory; 1786 BeamformerSharedMemoryLockKind dispatch_lock = BeamformerSharedMemoryLockKind_DispatchCompute; 1787 if (!sm->live_imaging_parameters.active && beamformer_shared_memory_take_lock(sm, (i32)dispatch_lock, 0)) 1788 { 1789 BeamformWork *work = beamform_work_queue_push(ctx->beamform_work_queue); 1790 if (work) { 1791 work->kind = BeamformerWorkKind_Compute; 1792 work->compute_context.view_plane = frame ? frame->view_plane_tag : 0; 1793 work->compute_context.parameter_block = parameter_block; 1794 beamform_work_queue_push_commit(ctx->beamform_work_queue); 1795 } 1796 } 1797 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1798 } 1799 1800 #include "ui.c" 1801 1802 function void 1803 beamformer_process_input_events(BeamformerCtx *ctx, BeamformerInput *input, 1804 BeamformerInputEvent *events, u32 event_count) 1805 { 1806 for (u32 index = 0; index < event_count; index++) { 1807 BeamformerInputEvent *event = events + index; 1808 switch (event->kind) { 1809 1810 // NOTE(rnp): ui will handle these 1811 case BeamformerInputEventKind_ButtonPress: 1812 case BeamformerInputEventKind_ButtonRelease: 1813 case BeamformerInputEventKind_MouseScroll: 1814 case BeamformerInputEventKind_WindowResize: 1815 {}break; 1816 1817 case BeamformerInputEventKind_ExecutableReload:{ 1818 ui_init(ctx, ctx->ui_arena); 1819 }break; 1820 1821 case BeamformerInputEventKind_FileEvent:{ 1822 BeamformerFileReloadContext *frc = event->file_watch_user_context; 1823 switch (frc->kind) { 1824 case BeamformerFileReloadKind_ComputeShader:{ 1825 for EachElement(ctx->compute_context.compute_plans, block) { 1826 BeamformerComputePlan *cp = ctx->compute_context.compute_plans[block]; 1827 for (u32 slot = 0; cp && slot < cp->pipeline.shader_count; slot++) { 1828 i32 shader_index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1829 if (beamformer_reloadable_shader_kinds[shader_index] == frc->shader_reload.shader) 1830 atomic_or_u32(&cp->dirty_programs, 1 << slot); 1831 } 1832 } 1833 1834 // TODO(rnp): track latest parameter block 1835 if (ctx->latest_frame) 1836 beamformer_queue_compute(ctx, ctx->latest_frame, 0); 1837 }break; 1838 1839 case BeamformerFileReloadKind_RenderShader:{ 1840 beamformer_reload_render_pipeline(frc->shader_reload.pipeline, frc->shader_reload.shader, ctx->arena); 1841 ctx->render_shader_updated = 1; 1842 }break; 1843 1844 InvalidDefaultCase; 1845 } 1846 }break; 1847 1848 InvalidDefaultCase; 1849 } 1850 } 1851 } 1852 1853 function void 1854 beamformer_panel_group_insert_at(BeamformerUIPanel *group, BeamformerUIPanel *tab, u64 new_child_index) 1855 { 1856 if (tab->parent) beamformer_ui_panel_unlink(tab); 1857 new_child_index = Min(new_child_index, group->child_count); 1858 1859 tab->parent = group; 1860 group->child_count++; 1861 if (group->kind == BeamformerPanelKind_TabGroup) group->u.tab_focus = tab; 1862 1863 BeamformerUIPanel *previous_sibling = new_child_index == 0 ? 0 : group->first_child; 1864 for (u64 child_index = 1; child_index < new_child_index; child_index++) 1865 previous_sibling = previous_sibling->next_sibling; 1866 1867 if (previous_sibling) { 1868 tab->previous_sibling = previous_sibling; 1869 tab->next_sibling = previous_sibling->next_sibling; 1870 if (tab->next_sibling) tab->next_sibling->previous_sibling = tab; 1871 previous_sibling->next_sibling = tab; 1872 if (previous_sibling == group->last_child) group->last_child = tab; 1873 } else { 1874 DLLInsertFirst(0, group->first_child, group->last_child, tab, next_sibling, previous_sibling); 1875 } 1876 } 1877 1878 BEAMFORMER_EXPORT void 1879 beamformer_frame_step(void *memory, BeamformerInput *input) 1880 { 1881 BeamformerCtx *ctx = beamformer_context = memory; 1882 beamformer_input = input; 1883 1884 u64 current_time = os_timer_count(); 1885 dt_for_frame = (f64)(current_time - ctx->frame_timestamp) / os_system_info()->timer_frequency; 1886 ctx->frame_timestamp = current_time; 1887 ctx->frame_index++; 1888 1889 coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats); 1890 1891 // NOTE(rnp): reset frame state 1892 { 1893 ctx->registers = &ctx->base_registers; 1894 swap(ctx->command_queues[0], ctx->command_queues[1]); 1895 zero_struct(ctx->command_queues + 0); 1896 //zero_struct(ctx->registers); 1897 arena_clear(beamformer_frame_arena()); 1898 } 1899 1900 beamformer_process_input_events(ctx, input, input->event_queue, input->event_count); 1901 1902 BeamformerSharedMemory *sm = ctx->shared_memory; 1903 u32 live_imaging_active = atomic_load_u32(&sm->live_imaging_parameters.active); 1904 if (live_imaging_active != ctx->live_imaging_active) { 1905 if (ctx->live_imaging_active) { 1906 if (ctx->auto_live_control_panel) { 1907 BeamformerUIPanel *parent = ctx->auto_live_control_panel->parent; 1908 beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)ctx->auto_live_control_panel); 1909 if (parent->child_count == 1) 1910 beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)parent); 1911 } 1912 } else { 1913 if (beamformer_registers()->live_controls) { 1914 beamformer_command(beamformer_command_infos[BeamformerCommandKind_FocusTab].string, 1915 .tree_node = beamformer_registers()->live_controls); 1916 } else { 1917 ctx->auto_live_control_panel = beamformer_ui_push_panel(0, BeamformerPanelKind_LiveImagingControls); 1918 beamformer_command(beamformer_command_infos[BeamformerCommandKind_SplitTree].string, 1919 .tree_node = (u64)ctx->auto_live_control_panel, 1920 .split_axis = Axis2_X, 1921 .split_left_tree = (u64)ui_context->tree, 1922 .split_right_tree = 0, 1923 .drop_target_tree = (u64)ui_context->tree); 1924 } 1925 ctx->live_imaging_active_frame = ctx->frame_index; 1926 } 1927 ctx->live_imaging_active = live_imaging_active; 1928 } 1929 1930 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_UploadRF)) 1931 os_wake_all_waiters(&ctx->upload_worker.sync_variable); 1932 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_DispatchCompute)) 1933 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1934 1935 beamformer_registers()->frame = (u64)(ctx->latest_frame - ctx->compute_context.backlog.frames); 1936 1937 beamformer_ui_frame(); 1938 1939 // NOTE(rnp): execute commands 1940 for (BeamformerCommandNode *node = ctx->command_queues[0].first; 1941 node; 1942 node = node == node->next ? 0 : node->next) 1943 { 1944 BeamformerRegistersScope() 1945 { 1946 memory_copy(beamformer_registers(), node->command.registers, sizeof(*node->command.registers)); 1947 BeamformerCommandKind kind = beamformer_command_kind_from_string(node->command.name); 1948 switch (kind) { 1949 InvalidDefaultCase; 1950 case BeamformerCommandKind_CloseTab:{ 1951 BeamformerUIPanel *tab = (BeamformerUIPanel *)beamformer_registers()->tree_node; 1952 ui_kill_panel(tab); 1953 }break; 1954 1955 case BeamformerCommandKind_FocusTab:{ 1956 BeamformerUIPanel *tab = (BeamformerUIPanel *)beamformer_registers()->tree_node; 1957 assert(tab->parent->kind == BeamformerPanelKind_TabGroup); 1958 tab->parent->u.tab_focus = tab; 1959 }break; 1960 1961 case BeamformerCommandKind_MoveTab:{ 1962 BeamformerUIPanel *move = (BeamformerUIPanel *)beamformer_registers()->tree_node; 1963 BeamformerUIPanel *group = (BeamformerUIPanel *)beamformer_registers()->drop_target_tree; 1964 BeamformerUIPanel *parent = move->parent; 1965 u64 new_child_index = beamformer_registers()->drop_child_index; 1966 beamformer_panel_group_insert_at(group, move, new_child_index); 1967 1968 if (move->kind == BeamformerPanelKind_LiveImagingControls) { 1969 beamformer_context->base_registers.v.live_controls = (u64)move; 1970 if (move == ctx->auto_live_control_panel) 1971 ctx->auto_live_control_panel = 0; 1972 } 1973 1974 if (parent->child_count == 0) 1975 beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)parent); 1976 }break; 1977 1978 case BeamformerCommandKind_OpenTab:{ 1979 BeamformerUIPanel *panel = (BeamformerUIPanel *)beamformer_registers()->tree_node; 1980 assert(panel->kind == BeamformerPanelKind_TabGroup); 1981 1982 BeamformerPanelKind new_panel_kind = beamformer_panel_kind_from_string(beamformer_registers()->string); 1983 beamformer_ui_push_panel(panel, new_panel_kind); 1984 }break; 1985 1986 case BeamformerCommandKind_SplitTree:{ 1987 BeamformerUIPanel *drag = (BeamformerUIPanel *)beamformer_registers()->tree_node; 1988 BeamformerUIPanel *left = (BeamformerUIPanel *)beamformer_registers()->split_left_tree; 1989 BeamformerUIPanel *right = (BeamformerUIPanel *)beamformer_registers()->split_right_tree; 1990 Axis2 axis = beamformer_registers()->split_axis; 1991 1992 BeamformerUIPanel *new_split = beamformer_ui_push_panel(0, BeamformerPanelKind_Split); 1993 BeamformerUIPanel *new_tab_group = beamformer_ui_push_panel(0, BeamformerPanelKind_TabGroup); 1994 beamformer_panel_group_insert_at(new_tab_group, drag, 0); 1995 1996 BeamformerUIPanel *target = 0; 1997 u32 target_child_index = 0; 1998 f32 new_split_pct = 0.5f; 1999 2000 if (left == 0 || right == 0) { 2001 // NOTE(rnp): split on edge of window 2002 target = left ? left : right; 2003 target_child_index = left ? 0 : 1; 2004 2005 if (target->kind == BeamformerPanelKind_TabGroup) { 2006 new_split->kind = BeamformerPanelKind_TabGroup; 2007 new_split->u.tab_focus = target->u.tab_focus; 2008 } 2009 2010 for (BeamformerUIPanel *child = target->last_child, *next; child; child = next) { 2011 next = child->previous_sibling; 2012 beamformer_panel_group_insert_at(new_split, child, 0); 2013 } 2014 2015 beamformer_panel_group_insert_at(target, new_tab_group, 0); 2016 } else if (((drag == left) && right->kind == BeamformerPanelKind_Split) || 2017 ((drag == right) && left->kind == BeamformerPanelKind_Split)) 2018 { 2019 // NOTE(rnp): split on internal split 2020 target = left == drag ? right : left; 2021 target_child_index = 1; 2022 new_split_pct = 1.f / 3.f; 2023 beamformer_panel_group_insert_at(new_split, new_tab_group, 0); 2024 beamformer_panel_group_insert_at(new_split, target->last_child, 1); 2025 } else { 2026 // NOTE(rnp): TabGroup Split 2027 target = left == drag ? right : left; 2028 target_child_index = left == drag ? 1 : 0; 2029 assert(target->kind == BeamformerPanelKind_TabGroup); 2030 2031 BeamformerUIPanel *focus = target->u.tab_focus; 2032 new_split->kind = BeamformerPanelKind_TabGroup; 2033 for (BeamformerUIPanel *child = target->last_child, *next; child; child = next) { 2034 next = child->previous_sibling; 2035 beamformer_panel_group_insert_at(new_split, child, 0); 2036 } 2037 new_split->u.tab_focus = focus; 2038 2039 beamformer_panel_group_insert_at(target, new_tab_group, 0); 2040 } 2041 2042 beamformer_panel_group_insert_at(target, new_split, target_child_index); 2043 if (target->kind == BeamformerPanelKind_Split) { 2044 new_split->u.split.axis = target->u.split.axis; 2045 new_split->u.split.fraction = target->u.split.fraction; 2046 } 2047 target->kind = BeamformerPanelKind_Split; 2048 target->u.split.axis = axis; 2049 target->u.split.fraction = new_split_pct; 2050 }break; 2051 2052 } 2053 } 2054 } 2055 2056 ctx->render_shader_updated = 0; 2057 }