beamformer_core.c (84789B)
1 /* See LICENSE for license details. */ 2 /* TODO(rnp): 3 * [ ]: backtrace dumping on SIGSEGV 4 * [ ]: cooperative shared memory loading in decode shader 5 * [ ]: refactor: save filter parameters with rest of parameters, whole slot thing is dumb 6 * [ ]: upload previously exported data for display. maybe this is a UI thing but doing it 7 * programatically would be nice. 8 * [ ]: Add interface for multi frame upload. RF upload already uses an offset into SM so 9 * that part works fine. We just need a way of specify a multi frame upload. (Data must 10 * be organized for simple offset access per frame). 11 * [ ]: refactor: do_compute should build its own "command graph" which tracks 12 * dependencies better. It is very important that unnecessary barriers are 13 * not placed between compute stages which requires knowledge of the entire 14 * graph. 15 * [ ]: refactor: replace UploadRF with just the scratch_rf_size variable, 16 * use below to spin wait in library 17 * [ ]: utilize umonitor/umwait (intel), monitorx/mwaitx (amd), and wfe/sev (aarch64) 18 * for power efficient low latency waiting 19 * [ ]: BeamformWorkQueue -> BeamformerWorkQueue 20 * [ ]: refactor: work queue needs a cleanup, we should only have a single one 21 * - that queue isn't really considered hot so a lock is probably fine 22 * [ ]: bug: reinit cuda on hot-reload 23 * 24 * [ ]: export for special frames/data 25 * - Color Map Data 26 * - Recursive Imaging Result 27 * - Incoherent sum 28 * 29 * [ ]: Tiled Array Handling 30 * [ ]: add tile count uv2 31 * [ ]: make xdc_transform an array 32 * [ ]: modify CPU DAS dispatch code to lookup xdc_transform based on tile index 33 * [ ]: modify CPU DAS dispatch code to set rf_element_offset based on tile index 34 * - need to check if this works for HERCULES or if the shader just needs to 35 * know about the extra tiles 36 * [ ]: write simple backpropagation code to optimize 2D tile position and 2D tilt 37 * (4 variables per tile). 38 * - two tests: 39 * 1. Maximize value of wire target 40 * 2. Maximize value of single cyst contrast 41 * 42 * [ ]: Recursive Imaging Handling 43 * [ ]: add array of image offsets to Compute Array Parameters 44 * [ ]: add array of weighting coeffiecents to Compute Array Parameters 45 * [ ]: Update 2D sum shader to use these 46 * 47 * [ ]: Power Doppler 48 * [ ]: also needs array of image offsets 49 * [ ]: make modified filter that grabs each sample from a different image offset 50 * - NOTE: this can't use existing striding mechanism because one image may 51 * be at the start of the ring buffer and another at the end of the 52 * ring buffer and the image size may not cleanly divide the ring buffer size. 53 * - Probably want this shader to just output the estimated power map directly. 54 * [ ]: make a modified render shader that takes two images: one structural and one that 55 * is used to index into a color map. (or second render pass that applies color overlay) 56 */ 57 58 #include "base_platform.h" 59 60 #if defined(BEAMFORMER_DEBUG) && !defined(BEAMFORMER_EXPORT) && OS_WINDOWS 61 #define BEAMFORMER_EXPORT __declspec(dllexport) 62 #endif 63 64 #include "beamformer_internal.h" 65 66 #define GPU_RESOURCE_HASH_TABLE_COUNT 256 67 68 typedef struct GPUResource GPUResource; 69 struct GPUResource { 70 str8 name; 71 u64 size; 72 u64 offset; 73 u64 alignment; 74 75 void *data; 76 77 u64 hash; 78 79 GPUResource *next; 80 GPUResource *hash_next, *hash_prev; 81 }; 82 typedef struct {GPUResource *first, *last;} GPUResourceHashBucket; 83 84 typedef struct { 85 Arena *arena; 86 u64 position; 87 88 GPUResource *resource_list; 89 GPUResourceHashBucket hash_table[GPU_RESOURCE_HASH_TABLE_COUNT]; 90 } GPUResourceBuilder; 91 92 read_only BeamformerFrame beamformer_nil_frame; 93 read_only BeamformerComputePlan beamformer_nil_compute_plan; 94 95 global BeamformerCtx *beamformer_context; 96 global BeamformerInput *beamformer_input; 97 global f32 dt_for_frame; 98 99 #define beamformer_frame_arena() (beamformer_context->frame_arenas[beamformer_context->frame_index % countof(beamformer_context->frame_arenas)]) 100 #define beamformer_registers() (&beamformer_context->registers->v) 101 #define beamformer_push_registers(...) beamformer_push_registers_(&(BeamformerRegisters){beamformer_registers_init_literal __VA_ARGS__}) 102 #define BeamformerRegistersScope(...) DeferLoop(beamformer_push_registers(__VA_ARGS__), beamformer_pop_registers()) 103 #define beamformer_command(name, ...) beamformer_push_command(name, &(BeamformerRegisters){beamformer_registers_init_literal __VA_ARGS__}) 104 105 function BeamformerRegisters * 106 beamformer_pop_registers(void) 107 { 108 BeamformerRegisters *result = &beamformer_context->registers->v; 109 SLLStackPop(beamformer_context->registers, next); 110 if (beamformer_context->registers == 0) 111 beamformer_context->registers = &beamformer_context->base_registers; 112 return result; 113 } 114 115 function BeamformerRegisters * 116 beamformer_push_registers_(BeamformerRegisters *registers) 117 { 118 BeamformerRegistersNode *node = push_struct(beamformer_frame_arena(), BeamformerRegistersNode); 119 BeamformerRegisters *result = &node->v; 120 memory_copy(result, registers, sizeof(node->v)); 121 SLLStackPush(beamformer_context->registers, node, next); 122 return result; 123 } 124 125 function void 126 beamformer_command_list_push_new(Arena *arena, BeamformerCommandList *commands, str8 name, BeamformerRegisters *registers) 127 { 128 BeamformerCommandNode *node = push_struct(arena, BeamformerCommandNode); 129 node->command.registers = push_struct_no_zero(arena, BeamformerRegisters); 130 node->command.name = push_str8(arena, name); 131 memory_copy(node->command.registers, registers, sizeof(*registers)); 132 DLLInsertLast(0, commands->first, commands->last, node, next, prev); 133 commands->count += 1; 134 } 135 136 function void 137 beamformer_push_command(str8 name, BeamformerRegisters *registers) 138 { 139 beamformer_command_list_push_new(beamformer_frame_arena(), beamformer_context->command_queues + 0, 140 name, registers); 141 } 142 143 function BeamformerCommandKind 144 beamformer_command_kind_from_string(str8 s) 145 { 146 BeamformerCommandKind result = BeamformerCommandKind_Nil; 147 for EachElement(beamformer_command_infos, it) { 148 if (str8_equal(beamformer_command_infos[it].string, s)) { 149 result = (BeamformerCommandKind)it; 150 break; 151 } 152 } 153 return result; 154 } 155 156 function BeamformerPanelKind 157 beamformer_panel_kind_from_string(str8 s) 158 { 159 BeamformerPanelKind result = BeamformerPanelKind_Nil; 160 for EachElement(beamformer_panel_infos, it) { 161 if (str8_equal(beamformer_panel_infos[it].string, s)) { 162 result = (BeamformerPanelKind)it; 163 break; 164 } 165 } 166 return result; 167 } 168 169 function BeamformerFrame * 170 beamformer_frame_from_index(u64 index) 171 { 172 BeamformerFrame *result = (BeamformerFrame *)&beamformer_nil_frame; 173 if (index < countof(beamformer_context->compute_context.backlog.frames)) { 174 BeamformerFrame *frame = beamformer_context->compute_context.backlog.frames + index; 175 if (frame->timeline_valid_value != 0) 176 result = frame; 177 } 178 return result; 179 } 180 181 function b32 182 beamformer_frame_valid(u64 index) 183 { 184 b32 result = beamformer_frame_from_index(index) != &beamformer_nil_frame; 185 return result; 186 } 187 188 function void 189 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block) 190 { 191 assert(block < countof(cc->compute_plans)); 192 BeamformerComputePlan *cp = cc->compute_plans[block]; 193 if (cp) { 194 gpu_buffer_release(&cp->gpu_temp_arena); 195 cc->compute_plans[block] = 0; 196 SLLPushFreelist(cp, cc->compute_plan_freelist); 197 } 198 } 199 200 function GPUResource * 201 gpu_resource_from_hash(GPUResourceBuilder *rb, u64 hash) 202 { 203 GPUResource *result = 0; 204 205 GPUResourceHashBucket *hb = rb->hash_table + (hash % GPU_RESOURCE_HASH_TABLE_COUNT); 206 for (GPUResource *r = hb->first; r; r = r->hash_next) { 207 if (hash == r->hash) { 208 result = r; 209 break; 210 } 211 } 212 213 return result; 214 } 215 216 typedef struct { 217 str8 name; 218 u64 align; 219 u64 size; 220 void *data; 221 } GPUResourcePushInfo; 222 #define gpu_resource_push(rb, t, count, ...) gpu_resource_push_(rb, (GPUResourcePushInfo){\ 223 .align = Max(alignof(t), 16), \ 224 .size = sizeof(t) * count, \ 225 __VA_ARGS__}) 226 227 function u32 228 gpu_resource_push_(GPUResourceBuilder *rb, GPUResourcePushInfo info) 229 { 230 assert(info.size > 0 && info.name.length > 0 && IsPowerOfTwo(info.align)); 231 232 u64 hash = u64_hash_from_str8(info.name); 233 GPUResource *r = gpu_resource_from_hash(rb, hash); 234 if (!r) { 235 r = push_struct(rb->arena, GPUResource); 236 GPUResourceHashBucket *hb = rb->hash_table + (hash % GPU_RESOURCE_HASH_TABLE_COUNT); 237 DLLInsert(0, hb->first, hb->last, r, hash_next, hash_prev); 238 SLLStackPush(rb->resource_list, r, next); 239 240 r->hash = hash; 241 r->name = info.name; 242 r->alignment = Max(16, info.align); 243 r->offset = AlignUpPowerOfTwo(rb->position, r->alignment); 244 r->size = info.size; 245 rb->position = r->offset + r->size; 246 } 247 248 u32 result = r->offset; 249 250 // NOTE(rnp): if this is a new resource and no data is provided it is likely 251 // a temporary GPU side buffer. if this is not a new resource and no data 252 // is provided then maybe it is shared and someone else will provide it 253 if (info.data) r->data = info.data; 254 255 return result; 256 } 257 258 function GPUResourceBuilder * 259 gpu_resource_build_begin(Arena *arena) 260 { 261 GPUResourceBuilder *result = push_struct(arena, GPUResourceBuilder); 262 result->arena = arena; 263 return result; 264 } 265 266 function void 267 gpu_resource_build_end(GPUResourceBuilder *rb, GPUBuffer *buffer) 268 { 269 u64 size = gpu_round_up_to_sync_size(rb->position, 64); 270 if (size != (u64)buffer->size) { 271 gpu_buffer_allocate(buffer, (GPUBufferAllocateInfo){ 272 .size = size, 273 .flags = GPUUsageFlag_HostWrite, 274 .label = push_str8_f(rb->arena, "GPU Temp Arena [%p]", buffer), 275 .timelines_used = (GPUTimeline []){GPUTimeline_Compute}, 276 .timeline_count = 1, 277 }); 278 } 279 280 ////////////////////////////////////// 281 // NOTE(rnp): upload data 282 u64 last_wait_value = 0; 283 for (GPUResource *r = rb->resource_list; r; r = r->next) 284 if (r->data) 285 last_wait_value = gpu_buffer_range_upload(buffer, r->data, r->offset, r->size, 0); 286 287 // TODO(rnp): cleanup this pointless stall 288 if (gpu_buffer_needs_sync(buffer)) 289 gpu_host_wait_timeline(GPUTimeline_Transfer, last_wait_value, -1ULL); 290 } 291 292 function BeamformerComputePlan * 293 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena) 294 { 295 assert(block < countof(cc->compute_plans)); 296 BeamformerComputePlan *result = cc->compute_plans[block]; 297 if (!result) { 298 result = SLLPopFreelist(cc->compute_plan_freelist); 299 if (!result) result = push_struct_no_zero(arena, BeamformerComputePlan); 300 zero_struct(result); 301 cc->compute_plans[block] = result; 302 303 result->ui_voxel_transform = m4_identity(); 304 } 305 return result; 306 } 307 308 function BeamformerFilter * 309 beamformer_filter_create(Arena *arena, BeamformerFilterParameters fp) 310 { 311 BeamformerFilter *result = push_struct(arena, BeamformerFilter); 312 switch (fp.kind) { 313 case BeamformerFilterKind_Kaiser:{ 314 /* TODO(rnp): this should also support complex */ 315 /* TODO(rnp): implement this as an IFIR filter instead to reduce computation */ 316 result->data = kaiser_low_pass_filter(arena, fp.kaiser.cutoff_frequency, fp.sampling_frequency, 317 fp.kaiser.beta, (i32)fp.kaiser.length); 318 result->length = (i32)fp.kaiser.length; 319 result->time_delay = (f32)result->length / 2.0f / fp.sampling_frequency; 320 }break; 321 322 case BeamformerFilterKind_MatchedChirp:{ 323 typeof(fp.matched_chirp) *mc = &fp.matched_chirp; 324 f32 fs = fp.sampling_frequency; 325 result->length = (i32)(mc->duration * fs); 326 if (fp.complex) { 327 result->data = baseband_chirp(arena, mc->min_frequency, mc->max_frequency, fs, result->length, 1, 0.5f); 328 result->time_delay = complex_filter_first_moment(result->data, result->length, fs); 329 } else { 330 result->data = rf_chirp(arena, mc->min_frequency, mc->max_frequency, fs, result->length, 1); 331 result->time_delay = real_filter_first_moment(result->data, result->length, fs); 332 } 333 }break; 334 335 InvalidDefaultCase; 336 } 337 338 result->parameters = fp; 339 return result; 340 } 341 342 function iv3 343 das_valid_points(iv3 points) 344 { 345 iv3 result; 346 result.x = Max(points.x, 1); 347 result.y = Max(points.y, 1); 348 result.z = Max(points.z, 1); 349 return result; 350 } 351 352 function GPUBuffer * 353 beamformer_gpu_buffer_from_frame(BeamformerFrame *frame) 354 { 355 GPUBuffer *result = beamformer_context->compute_context.backlog.buffer; 356 if (!Between(frame->gpu_pointer, result->gpu_pointer, result->gpu_pointer + result->size)) { 357 result = 0; 358 BeamformerComputePlan *cp = beamformer_context->compute_context.compute_plans[frame->parameter_block]; 359 if (cp) { 360 result = &cp->gpu_temp_arena; 361 assert(Between(frame->gpu_pointer, result->gpu_pointer, result->gpu_pointer + result->size)); 362 } 363 } 364 return result; 365 } 366 367 function u64 368 beamformer_frame_byte_size(iv3 points, BeamformerDataKind kind) 369 { 370 u64 result = points.x * points.y * points.z * beamformer_data_kind_byte_size[kind]; 371 result = round_up_to(result, 64); 372 return result; 373 } 374 375 function u64 376 beamformer_incoherent_frame_byte_size(iv3 points, BeamformerDataKind kind) 377 { 378 u64 result = beamformer_frame_byte_size(points, kind) / beamformer_data_kind_element_count[kind]; 379 return result; 380 } 381 382 function BeamformerFrame * 383 beamformer_frame_next(BeamformerComputeContext *cc, iv3 output_points, b32 complex) 384 { 385 BeamformerFrameBacklog *bl = &cc->backlog; 386 387 BeamformerDataKind kind = complex ? BeamformerDataKind_Float32Complex : BeamformerDataKind_Float32; 388 u64 frame_size = beamformer_frame_byte_size(output_points, kind); 389 390 // TODO(rnp): handle this somewhat gracefully (even it produces garbled output) 391 assert(frame_size <= (u64)bl->buffer->size); 392 393 if (bl->next_offset > (u64)bl->buffer->size - frame_size) 394 bl->next_offset = 0; 395 396 u64 id = bl->counter++; 397 398 BeamformerFrame *result = bl->frames + (id % countof(bl->frames)); 399 atomic_store_u64(&result->timeline_valid_value, -1ULL); 400 result->id = id & U32_MAX; 401 result->gpu_pointer = bl->buffer->gpu_pointer + bl->next_offset; 402 result->points = output_points; 403 result->data_kind = kind; 404 405 bl->next_offset += frame_size; 406 407 return result; 408 } 409 410 function void 411 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info) 412 { 413 u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer); 414 t->buffer[index] = info; 415 } 416 417 function uv3 418 layout_for_output(iv3 points) 419 { 420 uv3 result = {{1, 1, 1}}; 421 422 b32 has_x = points.x > 1; 423 b32 has_y = points.y > 1; 424 b32 has_z = points.z > 1; 425 426 u32 subgroup_size = gpu_info()->subgroup_size; 427 u32 grid_3d_z_size = Max(1, subgroup_size / (4 * 4)); 428 u32 grid_2d_y_size = Max(1, subgroup_size / 8); 429 430 switch (iv3_dimension(points)) { 431 case 1:{ 432 if (has_x) result.x = subgroup_size; 433 if (has_y) result.y = subgroup_size; 434 if (has_z) result.z = subgroup_size; 435 }break; 436 437 case 2:{ 438 if (has_x && has_y) {result.x = 8; result.y = grid_2d_y_size;} 439 if (has_x && has_z) {result.x = 8; result.z = grid_2d_y_size;} 440 if (has_y && has_z) {result.y = 8; result.z = grid_2d_y_size;} 441 }break; 442 443 case 3:{result = (uv3){{4, 4, grid_3d_z_size}};}break; 444 445 InvalidDefaultCase; 446 } 447 448 return result; 449 } 450 451 function uv3 452 dispatch_for_output(uv3 layout, iv3 points) 453 { 454 uv3 result; 455 result.x = DivRoundUp(points.x, (u32)layout.x); 456 result.y = DivRoundUp(points.y, (u32)layout.y); 457 result.z = DivRoundUp(points.z, (u32)layout.z); 458 return result; 459 } 460 461 typedef struct BeamformerComputeGraphNode BeamformerComputeGraphNode; 462 struct BeamformerComputeGraphNode { 463 // NOTE(rnp): will be BeamformerShaderKind_Count for root node 464 BeamformerShaderKind kind; 465 466 // NOTE(rnp): when any of input or output stride is assigned it is assumed that 467 // the shader requires a fixed layout for input, output, or both. When two adjacent 468 // nodes require incompatible layouts the second pass over the graph will insert 469 // Reshape shaders in between. 470 BeamformerDataKind input_data_kind; 471 BeamformerDataLayout input_data_layout; 472 473 BeamformerDataKind output_data_kind; 474 BeamformerDataLayout output_data_layout; 475 476 uv3 rf_dimensions; 477 478 b32 requires_fp_input; 479 480 i32 user_pipeline_index; 481 482 BeamformerComputeGraphNode *prev; 483 BeamformerComputeGraphNode *next; 484 }; 485 486 typedef struct { 487 BeamformerComputeGraphNode *first; 488 BeamformerComputeGraphNode *last; 489 u64 count; 490 } BeamformerComputeGraph; 491 492 function b32 493 compute_plan_push_shader(BeamformerComputePlan *p, BeamformerComputeGraphNode *node, BeamformerShaderParameters *sp) 494 { 495 b32 result = 0; 496 if (p->pipeline.shader_count < countof(p->pipeline.shaders)) { 497 u32 index = p->pipeline.shader_count++; 498 p->pipeline.shaders[index] = node->kind; 499 zero_struct(p->shader_descriptors + index); 500 p->pipeline.parameters[index] = sp ? *sp : (BeamformerShaderParameters){0}; 501 502 p->shader_descriptors[index].input_data_kind = node->input_data_kind; 503 p->shader_descriptors[index].output_data_kind = node->output_data_kind; 504 505 result = 1; 506 } 507 return result; 508 } 509 510 function BeamformerComputeGraphNode * 511 push_compute_graph_node(BeamformerComputeGraph *graph, BeamformerShaderKind kind, Arena *arena) 512 { 513 BeamformerComputeGraphNode *result = push_struct(arena, BeamformerComputeGraphNode); 514 if (graph) { 515 if (graph->last) result->rf_dimensions = graph->last->rf_dimensions; 516 DLLInsertLast(0, graph->first, graph->last, result, next, prev); 517 graph->count++; 518 } 519 result->kind = kind; 520 result->user_pipeline_index = -1; 521 // NOTE(rnp): initially don't care data parameters 522 result->input_data_kind = BeamformerDataKind_Count; 523 result->output_data_kind = BeamformerDataKind_Count; 524 result->input_data_layout = BeamformerDataLayout_Count; 525 result->output_data_layout = BeamformerDataLayout_Count; 526 return result; 527 } 528 529 function uv3 530 beamformer_data_strides(BeamformerDataLayout layout, uv3 rf_dimensions) 531 { 532 assert(layout >= 0 && layout < BeamformerDataLayout_Count); 533 uv3 result = {0}; 534 BeamformerRFDimension x = beamformer_data_stride_indices[layout][0]; 535 BeamformerRFDimension y = beamformer_data_stride_indices[layout][1]; 536 BeamformerRFDimension z = beamformer_data_stride_indices[layout][2]; 537 result.E[z] = 1; 538 result.E[y] = rf_dimensions.E[z]; 539 result.E[x] = rf_dimensions.E[z] * rf_dimensions.E[y]; 540 return result; 541 } 542 543 function void 544 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, Arena *scratch) 545 { 546 b32 run_hilbert = 0; 547 b32 demodulate = 0; 548 549 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 550 switch (pb->pipeline.shaders[i]) { 551 case BeamformerShaderKind_Hilbert:{run_hilbert = 1;}break; 552 case BeamformerShaderKind_Demodulate:{demodulate = 1;}break; 553 default:{}break; 554 } 555 } 556 557 if (demodulate) run_hilbert = 0; 558 559 f32 sampling_frequency = pb->parameters.sampling_frequency; 560 u32 input_sample_count = pb->parameters.sample_count; 561 u32 acquisition_count = pb->parameters.acquisition_count; 562 u32 decimation_rate = Max(pb->parameters.decimation_rate, 1); 563 // NOTE(rnp): only relevant when demodulating 564 f32 samples_per_wave = (pb->parameters.sampling_frequency / 2.0f) / pb->parameters.demodulation_frequency; 565 if ((u32)pb->parameters.sampling_mode < (u32)BeamformerSamplingMode_Count) 566 samples_per_wave = beamformer_sampling_mode_samples_per_wave[pb->parameters.sampling_mode] / 2.0f; 567 568 cp->raw_channel_byte_stride = pb->parameters.sample_count * pb->parameters.acquisition_count 569 * beamformer_data_kind_byte_size[pb->pipeline.data_kind]; 570 571 BeamformerDataKind input_data_kind = pb->pipeline.data_kind; 572 if (demodulate) { 573 switch (input_data_kind) { 574 case BeamformerDataKind_Int16:{ input_data_kind = BeamformerDataKind_Int16Complex; }break; 575 case BeamformerDataKind_Float16:{input_data_kind = BeamformerDataKind_Float16Complex;}break; 576 case BeamformerDataKind_Float32:{input_data_kind = BeamformerDataKind_Float32Complex;}break; 577 default:{}break; 578 } 579 input_sample_count /= (2 * decimation_rate); 580 sampling_frequency /= (2 * decimation_rate); 581 } 582 583 cp->iq_pipeline = beamformer_data_kind_complex[input_data_kind] || run_hilbert; 584 585 BeamformerDataKind das_data_kind = cp->iq_pipeline ? BeamformerDataKind_Float32Complex 586 : BeamformerDataKind_Float32; 587 588 cp->channel_count = pb->parameters.channel_count; 589 u32 chunk_channel_count = Min(cp->channel_count, BeamformerChunkChannelCount); 590 591 cp->rf_size = input_sample_count * pb->parameters.acquisition_count * chunk_channel_count 592 * beamformer_data_kind_byte_size[das_data_kind]; 593 594 i64 buffer_size = PING_PONG_BUFFER_SLOTS * round_up_to(cp->rf_size, 64); 595 if (beamformer_context->compute_context.ping_pong_buffer.size < buffer_size) { 596 b32 cuda = cuda_supported(); 597 GPUBufferAllocateInfo allocate_info = { 598 .size = buffer_size, 599 .export = cuda ? &beamformer_context->compute_context.ping_pong_export_handle : 0, 600 .label = str8("PingPongBuffer"), 601 }; 602 gpu_buffer_allocate(&beamformer_context->compute_context.ping_pong_buffer, allocate_info); 603 604 // TODO(rnp): figure out how to share with CUDA 605 // IMPORTANT: on linux the handle is returned to os and should be cleared after import 606 // see usage of glImportMemoryFdEXT and surrounding code in ui.c for examples 607 if (cuda) { 608 } 609 } 610 611 read_only BeamformerDataKind data_kind_to_element_kind[] = { 612 [BeamformerDataKind_Int16] = BeamformerDataKind_Float16, 613 [BeamformerDataKind_Float16] = BeamformerDataKind_Float16, 614 [BeamformerDataKind_Float32] = BeamformerDataKind_Float32, 615 [BeamformerDataKind_Int16Complex] = BeamformerDataKind_Float16, 616 [BeamformerDataKind_Float16Complex] = BeamformerDataKind_Float16, 617 [BeamformerDataKind_Float32Complex] = BeamformerDataKind_Float32, 618 }; 619 620 read_only BeamformerDataKind data_kind_to_fp_kind[] = { 621 [BeamformerDataKind_Int16] = BeamformerDataKind_Float16, 622 [BeamformerDataKind_Float16] = BeamformerDataKind_Float16, 623 [BeamformerDataKind_Float32] = BeamformerDataKind_Float32, 624 [BeamformerDataKind_Int16Complex] = BeamformerDataKind_Float16Complex, 625 [BeamformerDataKind_Float16Complex] = BeamformerDataKind_Float16Complex, 626 [BeamformerDataKind_Float32Complex] = BeamformerDataKind_Float32Complex, 627 }; 628 629 ////////////////////////////////////// 630 // NOTE(rnp): First Pass: build initial graph and insert hard layout constraints 631 BeamformerComputeGraph graph = {0}; 632 BeamformerComputeGraphNode *root_node = push_compute_graph_node(&graph, BeamformerShaderKind_Count, scratch); 633 root_node->input_data_kind = input_data_kind; 634 root_node->input_data_layout = BeamformerDataLayout_ChannelEventSample; 635 root_node->output_data_kind = input_data_kind; 636 root_node->output_data_layout = BeamformerDataLayout_ChannelEventSample; 637 638 root_node->rf_dimensions.E[BeamformerRFDimension_Channels] = chunk_channel_count; 639 root_node->rf_dimensions.E[BeamformerRFDimension_Events] = pb->parameters.acquisition_count; 640 root_node->rf_dimensions.E[BeamformerRFDimension_Samples] = pb->parameters.sample_count; 641 642 for EachIndex(pb->pipeline.shader_count, it) { 643 // NOTE(rnp): skip unnecessary shaders 644 switch (pb->pipeline.shaders[it]) { 645 case BeamformerShaderKind_Hilbert:{if (!run_hilbert) continue;}break; 646 647 case BeamformerShaderKind_Decode:{ 648 if (pb->parameters.decode_mode == BeamformerDecodeMode_None) 649 continue; 650 }break; 651 652 case BeamformerShaderKind_Sum: 653 case BeamformerShaderKind_MinMax: 654 { 655 // NOTE(rnp): currently unsupported 656 continue; 657 }break; 658 659 default:{}break; 660 } 661 662 BeamformerComputeGraphNode *node = push_compute_graph_node(&graph, pb->pipeline.shaders[it], scratch); 663 node->user_pipeline_index = (i32)it; 664 switch (pb->pipeline.shaders[it]) { 665 case BeamformerShaderKind_Demodulate:{ 666 node->rf_dimensions.E[BeamformerRFDimension_Samples] /= (2 * decimation_rate); 667 }break; 668 669 case BeamformerShaderKind_Decode:{ 670 b32 low_precision = beamformer_data_kind_element_size[input_data_kind] < 4; 671 b32 use_coop_matrix = gpu_info()->cooperative_matrix && 672 low_precision && 673 (acquisition_count % 16 == 0) && 674 (chunk_channel_count % 16 == 0); 675 676 node->requires_fp_input = 1; 677 678 // NOTE(rnp): fixed input layout required for reasonable performance 679 node->input_data_layout = BeamformerDataLayout_SampleChannelEvent; 680 681 if (use_coop_matrix) { 682 node->input_data_kind = BeamformerDataKind_Float16; 683 node->output_data_kind = data_kind_to_element_kind[das_data_kind]; 684 node->output_data_layout = node->input_data_layout; 685 } 686 }break; 687 688 case BeamformerShaderKind_DAS:{ 689 // NOTE(rnp): if we aren't decoding we can let the previous stage's 690 // data kind pass through as long as it is floating point 691 node->requires_fp_input = 1; 692 if (pb->parameters.decode_mode != BeamformerDecodeMode_None) 693 node->input_data_kind = das_data_kind; 694 695 node->input_data_layout = BeamformerDataLayout_ChannelEventSample; 696 if (pb->parameters.acquisition_kind == BeamformerAcquisitionKind_RCA_TPW || 697 pb->parameters.acquisition_kind == BeamformerAcquisitionKind_RCA_VLS) 698 { 699 node->input_data_layout = BeamformerDataLayout_EventChannelSample; 700 } 701 702 node->output_data_layout = BeamformerDataLayout_Image; 703 node->output_data_kind = das_data_kind; 704 705 // NOTE(rnp): insert implicit CoherencyWeighting node 706 if (pb->parameters.coherency_weighting) 707 node = push_compute_graph_node(&graph, BeamformerShaderKind_CoherencyWeighting, scratch); 708 }break; 709 710 default:{}break; 711 } 712 } 713 714 ////////////////////////////////////// 715 // NOTE(rnp): Second Pass: resolve layout constraints 716 for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) { 717 b32 needs_reshape = 0; 718 719 // NOTE(rnp): data strides 720 { 721 b32 input_dont_care = node->input_data_layout == BeamformerDataLayout_Count; 722 b32 prev_output_dont_care = node->prev->output_data_layout == BeamformerDataLayout_Count; 723 724 if (prev_output_dont_care && !input_dont_care) 725 node->prev->output_data_layout = node->input_data_layout; 726 727 if (!prev_output_dont_care && input_dont_care) 728 node->input_data_layout = node->prev->output_data_layout; 729 730 if (prev_output_dont_care && input_dont_care) 731 node->input_data_layout = node->prev->output_data_layout = node->prev->input_data_layout; 732 733 needs_reshape |= node->input_data_layout != node->prev->output_data_layout; 734 } 735 736 // NOTE(rnp): data kinds 737 { 738 b32 input_dont_care = node->input_data_kind == BeamformerDataKind_Count; 739 b32 prev_output_dont_care = node->prev->output_data_kind == BeamformerDataKind_Count; 740 741 if (prev_output_dont_care && !input_dont_care) 742 node->prev->output_data_kind = node->input_data_kind; 743 744 if (!prev_output_dont_care && input_dont_care) 745 node->input_data_kind = node->prev->output_data_kind; 746 747 if (prev_output_dont_care && input_dont_care) 748 node->input_data_kind = node->prev->output_data_kind = node->prev->input_data_kind; 749 750 if (node->requires_fp_input) { 751 node->input_data_kind = data_kind_to_fp_kind[node->input_data_kind]; 752 if (prev_output_dont_care) 753 node->prev->output_data_kind = node->input_data_kind; 754 } 755 756 needs_reshape |= node->input_data_kind != node->prev->output_data_kind; 757 } 758 759 // NOTE(rnp): insert reshape if needed 760 if (needs_reshape) { 761 BeamformerComputeGraphNode *new = push_compute_graph_node(0, BeamformerShaderKind_Reshape, scratch); 762 BeamformerComputeGraphNode *last = node->prev; 763 DLLInsertLast(0, node, last, new, next, prev); 764 graph.count++; 765 new->rf_dimensions = new->prev->rf_dimensions; 766 new->input_data_kind = new->prev->output_data_kind; 767 new->input_data_layout = new->prev->output_data_layout; 768 new->output_data_kind = new->next->input_data_kind; 769 new->output_data_layout = new->next->input_data_layout; 770 } 771 } 772 773 // NOTE(rnp): ensure last node descriptor gets proper values for output data kind 774 if (graph.last->output_data_kind == BeamformerDataKind_Count) 775 graph.last->output_data_kind = graph.last->input_data_kind; 776 777 f32 time_offset = pb->parameters.time_offset; 778 u32 subgroup_size = gpu_info()->subgroup_size; 779 780 cp->first_image_shader_index = 0; 781 cp->pipeline.shader_count = 0; 782 783 GPUResourceBuilder *resource_builder = gpu_resource_build_begin(scratch); 784 for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) { 785 assert(node->prev->output_data_kind == node->input_data_kind); 786 assert(node->prev->output_data_layout == node->input_data_layout); 787 788 BeamformerShaderParameters *sp = 0; 789 if (node->user_pipeline_index >= 0) 790 sp = pb->pipeline.parameters + node->user_pipeline_index; 791 792 if (compute_plan_push_shader(cp, node, sp)) { 793 BeamformerShaderDescriptor *sd = cp->shader_descriptors + cp->pipeline.shader_count - 1; 794 795 uv3 output_stride = beamformer_data_strides(node->output_data_layout, node->rf_dimensions); 796 uv3 input_stride = beamformer_data_strides(node->input_data_layout, node->prev->rf_dimensions); 797 798 switch (node->kind) { 799 case BeamformerShaderKind_Decode:{ 800 BeamformerDecodeBakeParameters *db = &sd->bake.Decode; 801 802 u32 decode_sample_count = node->rf_dimensions.E[BeamformerRFDimension_Samples]; 803 db->DecodeMode = pb->parameters.decode_mode; 804 db->TransmitCount = node->rf_dimensions.E[BeamformerRFDimension_Events]; 805 db->ChunkChannelCount = node->rf_dimensions.E[BeamformerRFDimension_Channels]; 806 807 // NOTE(rnp): ignored when using coop matrices 808 db->OutputSampleStride = output_stride.E[BeamformerRFDimension_Samples]; 809 db->OutputChannelStride = output_stride.E[BeamformerRFDimension_Channels]; 810 db->OutputTransmitStride = output_stride.E[BeamformerRFDimension_Events]; 811 812 db->ToProcess = 1; 813 814 b32 use_coop_matrix = gpu_info()->cooperative_matrix && 815 node->input_data_kind == BeamformerDataKind_Float16 && 816 (db->TransmitCount % 16 == 0) && 817 (chunk_channel_count % 16 == 0); 818 if (use_coop_matrix) { 819 // TODO(rnp): shared memory for larger sizes 820 sd->layout = (uv3){{subgroup_size, 1, 1}}; 821 822 if (demodulate) 823 decode_sample_count *= 2; 824 825 sd->compile_flags |= BeamformerDecodeCompileFlags_CooperativeMatrix; 826 db->CooperativeMatrixM = 16; 827 db->CooperativeMatrixN = 16; 828 db->CooperativeMatrixK = 16; 829 830 sd->dispatch.x = db->TransmitCount / db->CooperativeMatrixN; 831 sd->dispatch.y = chunk_channel_count / db->CooperativeMatrixM; 832 sd->dispatch.z = decode_sample_count; 833 } else if (db->TransmitCount > 40) { 834 sd->compile_flags |= BeamformerDecodeCompileFlags_UseSharedMemory; 835 836 if (db->TransmitCount == 48) 837 db->ToProcess = db->TransmitCount / 16; 838 839 b32 use_16x = db->TransmitCount == 48 || db->TransmitCount == 80 || 840 db->TransmitCount == 96 || db->TransmitCount == 160; 841 sd->layout.x = use_16x ? 16 : 32; 842 sd->layout.y = 4; 843 sd->layout.z = 1; 844 845 sd->dispatch.x = DivRoundUp(db->TransmitCount / db->ToProcess, sd->layout.x); 846 sd->dispatch.y = DivRoundUp(db->ChunkChannelCount, sd->layout.y); 847 sd->dispatch.z = DivRoundUp(decode_sample_count, sd->layout.z); 848 } else { 849 /* NOTE(rnp): register caching. using more threads will cause the compiler to do 850 * contortions to avoid spilling registers. using less gives higher performance */ 851 sd->layout = (uv3){{subgroup_size / 2, 1, 1}}; 852 853 sd->dispatch.x = DivRoundUp(decode_sample_count, sd->layout.x); 854 sd->dispatch.y = DivRoundUp(db->ChunkChannelCount, sd->layout.y); 855 sd->dispatch.z = 1; 856 } 857 858 sd->uses_heap = 1; 859 u32 order = db->TransmitCount; 860 db->Hadamard = gpu_resource_push(resource_builder, f16, order * order, 861 .data = make_hadamard_transpose(scratch, order, use_coop_matrix), 862 .name = str8("hadamard")); 863 }break; 864 865 case BeamformerShaderKind_Demodulate: 866 case BeamformerShaderKind_Filter: 867 { 868 b32 demod = node->kind == BeamformerShaderKind_Demodulate; 869 BeamformerFilter *f = beamformer_filter_create(scratch, cp->filter_parameters[sp->filter_slot]); 870 871 sd->compile_flags |= BeamformerFilterCompileFlags_Demodulate * demod; 872 sd->compile_flags |= BeamformerFilterCompileFlags_ComplexFilter * f->parameters.complex; 873 874 time_offset += f->time_delay; 875 876 BeamformerFilterBakeParameters *fb = &sd->bake.Filter; 877 878 sd->uses_heap = 1; 879 fb->FilterLength = (u32)f->length; 880 fb->FilterCoefficients = gpu_resource_push(resource_builder, f32, f->length * (f->parameters.complex ? 2 : 1), 881 .data = f->data, 882 .name = push_str8_f(scratch, "filter_%u", sp->filter_slot)); 883 884 fb->SampleCount = node->rf_dimensions.E[BeamformerRFDimension_Samples]; 885 fb->DecimationRate = demod ? decimation_rate : 1; 886 887 b32 deinterleave = beamformer_data_kind_complex[node->input_data_kind] && 888 !beamformer_data_kind_complex[node->output_data_kind]; 889 if (deinterleave) 890 fb->BatchSampleCount = node->rf_dimensions.x * node->rf_dimensions.y * node->rf_dimensions.z; 891 892 fb->OutputSampleStride = output_stride.E[BeamformerRFDimension_Samples]; 893 fb->OutputChannelStride = output_stride.E[BeamformerRFDimension_Channels]; 894 fb->OutputTransmitStride = output_stride.E[BeamformerRFDimension_Events]; 895 896 fb->InputSampleStride = input_stride.E[BeamformerRFDimension_Samples]; 897 fb->InputChannelStride = input_stride.E[BeamformerRFDimension_Channels]; 898 fb->InputTransmitStride = input_stride.E[BeamformerRFDimension_Events]; 899 900 /* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating 901 * between sampling the I portion and the Q portion of an IQ signal. Therefore there 902 * is an implicit decimation factor of 2 which must always be included. All code here 903 * assumes that the signal was sampled in such a way that supports this operation. 904 * To recover IQ[n] from the sampled data (RF[n]) we do the following: 905 * I[n] = RF[n] 906 * Q[n] = RF[n + 1] 907 * IQ[n] = I[n] - j*Q[n] 908 */ 909 fb->SamplesPerWave = samples_per_wave; 910 911 sd->layout = (uv3){{subgroup_size, 1, 1}}; 912 sd->dispatch.x = DivRoundUp(node->rf_dimensions.E[BeamformerRFDimension_Samples], sd->layout.x); 913 sd->dispatch.y = DivRoundUp(node->rf_dimensions.E[BeamformerRFDimension_Channels], sd->layout.y); 914 sd->dispatch.z = DivRoundUp(node->rf_dimensions.E[BeamformerRFDimension_Events], sd->layout.z); 915 }break; 916 917 case BeamformerShaderKind_DAS:{ 918 cp->first_image_shader_index = cp->pipeline.shader_count; 919 920 BeamformerDASBakeParameters *db = &sd->bake.DAS; 921 db->SamplingFrequency = sampling_frequency; 922 db->SamplesPerWave = samples_per_wave / decimation_rate; 923 db->SpeedOfSound = pb->parameters.speed_of_sound; 924 db->TimeOffset = time_offset; 925 db->FNumber = pb->parameters.f_number; 926 db->AcquisitionKind = pb->parameters.acquisition_kind; 927 db->SampleCount = node->rf_dimensions.E[BeamformerRFDimension_Samples]; 928 db->AcquisitionCount = node->rf_dimensions.E[BeamformerRFDimension_Events]; 929 db->ChunkChannelCount = node->rf_dimensions.E[BeamformerRFDimension_Channels]; 930 db->ChannelByteStride = input_stride.E[BeamformerRFDimension_Channels] 931 * beamformer_data_kind_byte_size[node->input_data_kind]; 932 db->AcquisitionByteStride = input_stride.E[BeamformerRFDimension_Events] 933 * beamformer_data_kind_byte_size[node->input_data_kind]; 934 935 // TODO(rnp): pass these in 936 db->ReceiveChannelCount = pb->parameters.channel_count; 937 db->TransmitChannelCount = pb->parameters.channel_count; 938 db->ReceiveTileCount = 1; 939 db->TransmitTileCount = 1; 940 941 db->InterpolationMode = pb->parameters.interpolation_mode; 942 db->ReadiGroupCount = pb->parameters.readi_group_count; 943 db->OutputSizeX = cp->output_points.x; 944 db->OutputSizeY = cp->output_points.y; 945 db->OutputSizeZ = cp->output_points.z; 946 db->TransmitReceiveOrientation = pb->parameters.transmit_receive_orientation; 947 948 cp->xdc_element_pitch = pb->parameters.xdc_element_pitch; 949 950 u32 id = pb->parameters.acquisition_kind; 951 952 u32 acquisition_kind_flags = beamformer_acquisition_kind_flags[id]; 953 b32 sparse = (acquisition_kind_flags & BeamformerAcquisitionKindFlags_Sparse) != 0; 954 b32 single_focus = pb->parameters.single_focus != 0; 955 b32 single_orientation = pb->parameters.single_orientation != 0; 956 957 sd->compile_flags |= BeamformerDASCompileFlags_CoherencyWeighting * (pb->parameters.coherency_weighting != 0); 958 sd->compile_flags |= BeamformerDASCompileFlags_Sparse * sparse; 959 960 sd->layout = layout_for_output(cp->output_points); 961 sd->dispatch = dispatch_for_output(sd->layout, cp->output_points); 962 963 sd->uses_heap = 1; 964 db->TransducerTransforms = gpu_resource_push(resource_builder, m4, db->TransmitTileCount * db->ReceiveTileCount, 965 .data = &pb->parameters.xdc_transform, 966 .name = str8("transducer_transforms")); 967 968 if (!single_focus) { 969 db->FocalVectors = gpu_resource_push(resource_builder, v2, db->AcquisitionCount, 970 .data = pb->focal_vectors, 971 .name = str8("focal_vectors")); 972 } 973 974 if (!single_orientation) { 975 db->TransmitReceiveOrientations = gpu_resource_push(resource_builder, u8, db->AcquisitionCount, 976 .data = pb->transmit_receive_orientations, 977 .name = str8("transmit_receive_orientations")); 978 } 979 980 if (sparse) { 981 db->SparseElements = gpu_resource_push(resource_builder, i16, db->AcquisitionCount, 982 .data = pb->sparse_elements, 983 .name = str8("sparse_elements")); 984 } 985 986 if (pb->parameters.coherency_weighting) { 987 db->IncoherentFrame = gpu_resource_push(resource_builder, u32, 0, 988 .size = beamformer_incoherent_frame_byte_size(cp->output_points, das_data_kind), 989 .name = str8("incoherent_buffer")); 990 } 991 992 cp->readi_group = pb->parameters.readi_group; 993 if (db->ReadiGroupCount > 1) { 994 u32 order = db->ReadiGroupCount; 995 db->Hadamard = gpu_resource_push(resource_builder, f16, order * order, 996 .data = make_hadamard_transpose(scratch, order, 0), 997 .name = str8("readi_hadamard")); 998 } 999 }break; 1000 1001 case BeamformerShaderKind_CoherencyWeighting:{ 1002 // NOTE(rnp): beamformed data is stored in linear order; making the layout 2D or 3D 1003 // here is just slower 1004 sd->layout = (uv3){{subgroup_size, 1, 1}}; 1005 sd->dispatch = dispatch_for_output(sd->layout, cp->output_points); 1006 1007 BeamformerCoherencyWeightingBakeParameters *cw = &sd->bake.CoherencyWeighting; 1008 cw->Scale = 1.f; 1009 cw->OutputVoxels = cp->output_points.x * cp->output_points.y * cp->output_points.z; 1010 cw->IncoherentSum = gpu_resource_push(resource_builder, u32, 0, 1011 .size = beamformer_incoherent_frame_byte_size(cp->output_points, das_data_kind), 1012 .name = str8("incoherent_buffer")); 1013 sd->uses_heap = 1; 1014 }break; 1015 1016 case BeamformerShaderKind_Reshape:{ 1017 BeamformerReshapeBakeParameters *rb = &sd->bake.Reshape; 1018 b32 deinterleave = beamformer_data_kind_complex[node->input_data_kind] && 1019 !beamformer_data_kind_complex[node->output_data_kind]; 1020 b32 interleave = !beamformer_data_kind_complex[node->input_data_kind] && 1021 beamformer_data_kind_complex[node->output_data_kind]; 1022 assert(interleave == 0 || (interleave != deinterleave)); 1023 sd->compile_flags |= BeamformerReshapeCompileFlags_Deinterleave * deinterleave; 1024 sd->compile_flags |= BeamformerReshapeCompileFlags_Interleave * interleave; 1025 1026 rb->InputStrideX = input_stride.x; 1027 rb->InputStrideY = input_stride.y; 1028 rb->InputStrideZ = input_stride.z; 1029 rb->OutputStrideX = output_stride.x; 1030 rb->OutputStrideY = output_stride.y; 1031 rb->OutputStrideZ = output_stride.z; 1032 1033 // NOTE(rnp): order doesn't really matter here but it must match the dispatch layout 1034 rb->SizeX = node->rf_dimensions.x; 1035 rb->SizeY = node->rf_dimensions.y; 1036 rb->SizeZ = node->rf_dimensions.z; 1037 1038 sd->layout.x = 1; 1039 sd->layout.z = Min(subgroup_size, rb->SizeZ); 1040 sd->layout.y = subgroup_size / sd->layout.z; 1041 1042 sd->dispatch.x = DivRoundUp(rb->SizeX, sd->layout.x); 1043 sd->dispatch.y = DivRoundUp(rb->SizeY, sd->layout.y); 1044 sd->dispatch.z = DivRoundUp(rb->SizeZ, sd->layout.z); 1045 }break; 1046 1047 default:{}break; 1048 1049 #if 0 1050 case BeamformerShaderKind_Sum:{ 1051 sd->bake.data_kind = BeamformerDataKind_Float32; 1052 if (cp->iq_pipeline) 1053 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 1054 1055 sd->layout = layout_for_output(cp->output_points); 1056 sd->dispatch = dispatch_for_output(sd->layout, cp->output_points); 1057 1058 commit = 1; 1059 }break; 1060 #endif 1061 1062 } 1063 } 1064 } 1065 1066 cp->pipeline.data_kind = input_data_kind; 1067 1068 if (cp->first_image_shader_index == 0) 1069 cp->first_image_shader_index = cp->pipeline.shader_count; 1070 1071 gpu_resource_build_end(resource_builder, &cp->gpu_temp_arena); 1072 } 1073 1074 function void 1075 stream_append_shader_header(Stream *s, i32 reloadable_index, u64 gpu_heap_pointer, BeamformerShaderDescriptor *sd, uv3 layout) 1076 { 1077 stream_append_str8(s, str8("#version 460 core\n\n" 1078 "#extension GL_EXT_buffer_reference : require\n" 1079 "#extension GL_EXT_shader_16bit_storage : require\n" 1080 "#extension GL_EXT_shader_explicit_arithmetic_types : require\n\n" 1081 "#define f32 float32_t\n" 1082 "#define f16 float16_t\n" 1083 "#define s32 int32_t\n" 1084 "#define u64 uint64_t\n" 1085 "#define u32 uint32_t\n" 1086 "#define s16 int16_t\n" 1087 "#define u16 uint16_t\n" 1088 "#define u8 uint8_t\n" 1089 "#define s32vec2 i32vec2\n" 1090 "#define s16vec2 i16vec2\n" 1091 "\n")); 1092 1093 i32 header_vector_length = beamformer_shader_header_vector_lengths[reloadable_index]; 1094 i32 *header_vector = (i32 *)beamformer_shader_header_vectors[reloadable_index]; 1095 for (i32 index = 0; index < header_vector_length; index++) 1096 stream_append_str8(s, beamformer_shader_global_header_strings[header_vector[index]]); 1097 1098 if (layout.x != 0) { 1099 stream_append_str8(s, str8("layout(local_size_x = ")); 1100 stream_append_u64(s, layout.x); 1101 stream_append_str8(s, str8(", local_size_y = ")); 1102 stream_append_u64(s, layout.y); 1103 stream_append_str8(s, str8(", local_size_z = ")); 1104 stream_append_u64(s, layout.z); 1105 stream_append_str8(s, str8(") in;\n\n")); 1106 } 1107 1108 { 1109 u32 max_length = 0; 1110 for EachElement(beamformer_data_kind_str8, it) 1111 max_length = Max(max_length, (u32)beamformer_data_kind_str8[it].length); 1112 1113 for EachElement(beamformer_data_kind_str8, it) { 1114 stream_append_str8s(s, str8("#define DataKind_"), beamformer_data_kind_str8[it]); 1115 stream_pad(s, ' ', max_length - beamformer_data_kind_str8[it].length + 1); 1116 stream_append_u64(s, it); 1117 stream_append_byte(s, '\n'); 1118 } 1119 stream_append_byte(s, '\n'); 1120 } 1121 1122 if (gpu_heap_pointer) { 1123 stream_append_str8(s, str8("#define HeapBase u64(0x")); 1124 stream_append_hex_u64(s, gpu_heap_pointer); 1125 stream_append_str8(s, str8("ul)\n")); 1126 } 1127 1128 if (sd) { 1129 BeamformerDataKind data_kinds[] = {sd->input_data_kind, sd->output_data_kind}; 1130 str8 line_prefixes[] = {str8_comp("Input"), str8_comp("Output")}; 1131 for EachElement(data_kinds, it) { 1132 if (data_kinds[it] != BeamformerDataKind_Count) { 1133 stream_append_str8s(s, str8("#define "), line_prefixes[it], str8("DataType "), 1134 beamformer_data_kind_glsl_type[data_kinds[it]], 1135 str8("\n#define "), line_prefixes[it], str8("DataKind DataKind_"), 1136 beamformer_data_kind_str8[data_kinds[it]], 1137 str8("\n#define "), line_prefixes[it], str8("DataKindByteSize ")); 1138 stream_append_u64(s, beamformer_data_kind_byte_size[data_kinds[it]]); 1139 stream_append_byte(s, '\n'); 1140 } 1141 } 1142 stream_append_byte(s, '\n'); 1143 1144 stream_append_str8(s, str8("#define CompileFlags (0x")); 1145 stream_append_hex_u64_width(s, sd->compile_flags, 8); 1146 stream_append_str8(s, str8(")\n")); 1147 1148 i32 struct_id = beamformer_base_shader_to_bake_struct_id[reloadable_index]; 1149 if (struct_id != -1) { 1150 const str8 *names = meta_struct_member_names_by_id[struct_id]; 1151 const MetaStructInfo *si = meta_struct_info_by_id + struct_id; 1152 const MetaStructMember *sm = meta_struct_members_by_id[struct_id]; 1153 for (u32 index = 0; index < si->member_count; index++) { 1154 str8 type = meta_kind_glsl_types[sm[index].type_id]; 1155 stream_append_str8(s, str8("layout(constant_id = ")); 1156 stream_append_u64(s, index); 1157 stream_append_str8s(s, str8(") const "), type, str8(" "), names[index], str8(" = "), type, str8("(1);\n")); 1158 } 1159 } 1160 } 1161 1162 if (!renderdoc_attached()) 1163 stream_append_str8(s, str8("\n\n#line 1\n")); 1164 } 1165 1166 function void 1167 beamformer_reload_pipeline(VulkanHandle *pipeline, BeamformerShaderReloadInfo *sris, u32 count, Arena *scratch) 1168 { 1169 assume(count <= 2); 1170 str8 paths[2]; 1171 VulkanPipelineCreateInfo infos[2]; 1172 1173 if (!BakeShaders) { 1174 for (u32 i = 0; i < count; i++) 1175 paths[i] = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), sris[i].filename_or_data); 1176 } 1177 1178 u32 push_constants_size = 0; 1179 for (u32 i = 0; i < count; i++) { 1180 Stream shader_stream = arena_stream(scratch); 1181 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[sris[i].shader]; 1182 if (i == 0) push_constants_size = beamformer_shader_push_constant_sizes[reloadable_index]; 1183 else assert(push_constants_size == beamformer_shader_push_constant_sizes[reloadable_index]); 1184 1185 stream_append_shader_header(&shader_stream, reloadable_index, sris[i].gpu_heap_pointer, 1186 sris[i].shader_descriptor, sris[i].layout); 1187 1188 str8 shader_text; 1189 if (BakeShaders) { 1190 stream_append_str8(&shader_stream, sris[i].filename_or_data); 1191 shader_text = arena_stream_commit_zero(scratch, &shader_stream); 1192 } else { 1193 str8 stream_data = arena_stream_commit(scratch, &shader_stream); 1194 str8 shader_data = os_read_entire_file(scratch, (c8 *)paths[i].data); 1195 // NOTE(rnp): kinda sucky but need to make sure these are a contiguous string 1196 shader_text = push_str8_from_parts(scratch, str8(""), stream_data, shader_data); 1197 } 1198 1199 infos[i].kind = sris[i].shader_kind; 1200 infos[i].text = shader_text; 1201 infos[i].name = beamformer_shader_names[sris[i].shader]; 1202 infos[i].specialization_data = sris[i].shader_descriptor ? &sris[i].shader_descriptor->bake : 0; 1203 infos[i].specialization_struct_id = beamformer_base_shader_to_bake_struct_id[reloadable_index]; 1204 1205 //str8 line = str8("---------------\n"); 1206 //str8 nl = str8("\n"); 1207 //os_console_log(line.data, line.length); 1208 //os_console_log(infos[i].name.data, infos[i].name.length); 1209 //os_console_log(nl.data, nl.length); 1210 //os_console_log(line.data, line.length); 1211 //os_console_log(infos[i].text.data, infos[i].text.length); 1212 //os_console_log(line.data, line.length); 1213 } 1214 1215 vk_pipeline_release(*pipeline); 1216 *pipeline = vk_pipeline(infos, count, push_constants_size); 1217 } 1218 1219 function void 1220 beamformer_reload_render_pipeline(VulkanHandle *pipeline, BeamformerShaderKind shader, Arena *scratch) 1221 { 1222 i32 index = beamformer_shader_reloadable_index_by_shader[shader]; 1223 BeamformerShaderReloadInfo infos[2] = { 1224 { 1225 .shader = shader, 1226 .shader_kind = beamformer_shader_primitive_is_vertex[index] ? VulkanShaderKind_Vertex : VulkanShaderKind_Mesh, 1227 .filename_or_data = BakeShaders ? beamformer_shader_data[index][0] 1228 : beamformer_reloadable_shader_files[index][0], 1229 }, 1230 { 1231 .shader = shader, 1232 .shader_kind = VulkanShaderKind_Fragment, 1233 .filename_or_data = BakeShaders ? beamformer_shader_data[index][1] 1234 : beamformer_reloadable_shader_files[index][1], 1235 }, 1236 }; 1237 beamformer_reload_pipeline(pipeline, infos, countof(infos), scratch); 1238 } 1239 1240 function void 1241 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena *scratch) 1242 { 1243 BeamformerParameterBlock *pb; 1244 DeferLoop(pb = beamformer_parameter_block_lock(ctx->shared_memory, block, -1), 1245 beamformer_parameter_block_unlock(ctx->shared_memory, block)) 1246 for EachBit(pb->region_update_flags, region) 1247 { 1248 pb->region_update_flags &= ~(1ul << region); 1249 switch (region) { 1250 case BeamformerParameterDirtyFlag_NotifyUI:{ 1251 atomic_store_u32(&ctx->ui_dirty_parameter_blocks, 1u << block); 1252 cp->ui_voxel_transform = m4_identity(); 1253 }break; 1254 1255 case BeamformerParameterDirtyFlag_Parameters:{ 1256 cp->output_points = das_valid_points(pb->parameters.output_points.xyz); 1257 cp->average_frames = pb->parameters.output_points.E[3]; 1258 1259 plan_compute_pipeline(cp, pb, scratch); 1260 1261 b32 heap_pointer_updated = cp->last_gpu_heap_pointer != cp->gpu_temp_arena.gpu_pointer; 1262 cp->last_gpu_heap_pointer = cp->gpu_temp_arena.gpu_pointer; 1263 1264 for (u32 shader_slot = 0; shader_slot < cp->pipeline.shader_count; shader_slot++) { 1265 u128 hash = u128_hash_from_data(cp->shader_descriptors + shader_slot, sizeof(BeamformerShaderDescriptor)); 1266 if (!u128_equal(hash, cp->shader_hashes[shader_slot]) || 1267 (cp->shader_descriptors[shader_slot].uses_heap && heap_pointer_updated)) 1268 { 1269 cp->dirty_programs |= 1 << shader_slot; 1270 } 1271 cp->shader_hashes[shader_slot] = hash; 1272 } 1273 1274 cp->acquisition_count = pb->parameters.acquisition_count; 1275 cp->acquisition_kind = pb->parameters.acquisition_kind; 1276 cp->contrast_mode = pb->parameters.contrast_mode; 1277 }break; 1278 } 1279 } 1280 } 1281 1282 function void 1283 do_compute_shader(GPUCommandList cmd, BeamformerComputePlan *cp, BeamformerFrame *frame, 1284 u64 input_pointer, u64 output_pointer, u32 shader_slot, u32 channel_offset) 1285 { 1286 BeamformerComputeContext *cc = &beamformer_context->compute_context; 1287 1288 uv3 dispatch = cp->shader_descriptors[shader_slot].dispatch; 1289 1290 gpu_command_bind_pipeline(cmd, cp->vulkan_pipelines[shader_slot]); 1291 1292 switch (cp->pipeline.shaders[shader_slot]) { 1293 1294 case BeamformerShaderKind_Decode:{ 1295 BeamformerDecodePushConstants pc = { 1296 .rf_buffer = input_pointer, 1297 .output_buffer = output_pointer, 1298 }; 1299 1300 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1301 gpu_command_dispatch_compute(cmd, dispatch); 1302 1303 cc->ping_pong_input_index = !cc->ping_pong_input_index; 1304 }break; 1305 1306 case BeamformerShaderKind_Hilbert:{ 1307 //cuda_hilbert(input_index, output_index); 1308 //cc->ping_pong_input_index = !cc->ping_pong_input_index; 1309 }break; 1310 1311 case BeamformerShaderKind_Filter: 1312 case BeamformerShaderKind_Demodulate: 1313 { 1314 BeamformerFilterPushConstants pc = { 1315 .input_buffer = input_pointer, 1316 .output_buffer = output_pointer, 1317 }; 1318 1319 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1320 gpu_command_dispatch_compute(cmd, dispatch); 1321 1322 cc->ping_pong_input_index = !cc->ping_pong_input_index; 1323 }break; 1324 1325 case BeamformerShaderKind_DAS:{ 1326 BeamformerDASPushConstants pc = { 1327 .xdc_element_pitch = cp->xdc_element_pitch, 1328 .rf_data = input_pointer, 1329 .output_frame = frame->gpu_pointer, 1330 .channel_offset = channel_offset, 1331 .focus_depth = frame->focus_depth, 1332 .steering_angle = frame->steering_angle, 1333 .tx_rx_orientation = frame->transmit_receive_orientation, 1334 .readi_group = cp->readi_group, 1335 }; 1336 memory_copy(pc.voxel_transform.E, frame->voxel_transform.E, sizeof(pc.voxel_transform)); 1337 1338 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1339 gpu_command_dispatch_compute(cmd, dispatch); 1340 }break; 1341 1342 case BeamformerShaderKind_CoherencyWeighting:{ 1343 BeamformerCoherencyWeightingPushConstants pc = {.coherent_sum = frame->gpu_pointer}; 1344 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1345 gpu_command_dispatch_compute(cmd, dispatch); 1346 }break; 1347 1348 case BeamformerShaderKind_Reshape:{ 1349 BeamformerDataKind input_data_kind = cp->shader_descriptors[shader_slot].input_data_kind; 1350 BeamformerReshapeBakeParameters *rb = &cp->shader_descriptors[shader_slot].bake.Reshape; 1351 BeamformerReshapePushConstants pc = { 1352 .left_input_buffer = input_pointer, 1353 .right_input_buffer = input_pointer + rb->SizeX * rb->SizeY * rb->SizeZ 1354 * beamformer_data_kind_byte_size[input_data_kind], 1355 .output_buffer = output_pointer, 1356 }; 1357 1358 gpu_command_push_constants(cmd, 0, sizeof(pc), &pc); 1359 gpu_command_dispatch_compute(cmd, dispatch); 1360 1361 cc->ping_pong_input_index = !cc->ping_pong_input_index; 1362 }break; 1363 1364 // NOTE(rnp): invalid stages should be filtered in planning phase 1365 InvalidDefaultCase; 1366 } 1367 1368 #if 0 1369 switch (shader) { 1370 case BeamformerShaderKind_MinMax:{ 1371 for (u32 i = 1; i < frame->image.mip_map_levels; i++) { 1372 glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 1373 glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F); 1374 glProgramUniform1i(program, MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i); 1375 1376 u32 width = (u32)frame->dim.x >> i; 1377 u32 height = (u32)frame->dim.y >> i; 1378 u32 depth = (u32)frame->dim.z >> i; 1379 glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32)); 1380 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1381 } 1382 }break; 1383 case BeamformerShaderKind_Sum:{ 1384 u32 aframe_index = ctx->averaged_frame_index % countof(ctx->averaged_frames); 1385 BeamformerFrame *aframe = ctx->averaged_frames + aframe_index; 1386 aframe->id = ctx->averaged_frame_index; 1387 atomic_store_u32(&aframe->ready_to_present, 0); 1388 /* TODO(rnp): hack we need a better way of specifying which frames to sum; 1389 * this is fine for rolling averaging but what if we want to do something else */ 1390 assert(frame >= ctx->beamform_frames); 1391 assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames)); 1392 u32 base_index = (u32)(frame - ctx->beamform_frames); 1393 u32 to_average = (u32)cp->average_frames; 1394 u32 frame_count = 0; 1395 u32 *in_textures = push_array(&arena, u32, BeamformerMaxBacklogFrames); 1396 ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average); 1397 for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi)) 1398 in_textures[frame_count++] = it->texture; 1399 1400 assert(to_average == frame_count); 1401 1402 glProgramUniform1f(program, SUM_PRESCALE_UNIFORM_LOC, 1 / (f32)frame_count); 1403 /* NOTE: zero output before summing */ 1404 glClearTexImage(aframe->texture, 0, GL_RED, GL_FLOAT, 0); 1405 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 1406 1407 glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F); 1408 for (u32 i = 0; i < in_texture_count; i++) { 1409 glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 1410 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 1411 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1412 } 1413 1414 memory_copy(aframe->voxel_transform.E, frame->voxel_transform.E, sizeof(frame->voxel_transform)); 1415 aframe->compound_count = frame->compound_count; 1416 aframe->acquisition_kind = frame->acquisition_kind; 1417 }break; 1418 } 1419 #endif 1420 } 1421 1422 function void 1423 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena) 1424 { 1425 BeamformerComputeContext * cs = &ctx->compute_context; 1426 BeamformerSharedMemory * sm = ctx->shared_memory; 1427 1428 for (BeamformWork *work = beamform_work_queue_pop(q); 1429 work; 1430 beamform_work_queue_pop_commit(q), work = beamform_work_queue_pop(q)) 1431 { 1432 switch (work->kind) { 1433 1434 case BeamformerWorkKind_ExportBuffer:{ 1435 /* TODO(rnp): better way of handling DispatchCompute barrier */ 1436 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute); 1437 BeamformerExportContext *ec = &work->export_context; 1438 switch (ec->kind) { 1439 case BeamformerExportKind_BeamformedData:{ 1440 BeamformerFrameBacklog *bl = &ctx->compute_context.backlog; 1441 u32 req_count = Clamp(ec->count, 1, bl->counter); 1442 u32 frame_idx = bl->counter - req_count; 1443 1444 u8 *sm_output = beamformer_shared_memory_allocation_pointer(sm, ec->memory); 1445 u64 total_size = beamformer_shared_memory_allocation_size(sm, ec->memory); 1446 1447 u64 exported_size = 0; 1448 for (u32 export_count = 0; export_count < req_count; export_count++, frame_idx++) { 1449 BeamformerFrame *f = bl->frames + frame_idx % countof(bl->frames); 1450 u64 frame_size = beamformer_frame_byte_size(f->points, f->data_kind); 1451 assert((frame_size & 63) == 0); 1452 // NOTE(tkh) we don't want to assume that all req_count frames are the same size, 1453 // so we either need to count the total size of all requested frames first or 1454 // just fill up as much as possible. 1455 if (exported_size + frame_size <= total_size) { 1456 u64 offset = f->gpu_pointer - bl->buffer->gpu_pointer; 1457 gpu_host_wait_timeline(GPUTimeline_Compute, f->timeline_valid_value, -1ULL); 1458 gpu_buffer_range_download(sm_output + exported_size, bl->buffer, offset, frame_size, 1); 1459 exported_size += frame_size; 1460 } 1461 } 1462 }break; 1463 1464 case BeamformerExportKind_Stats:{ 1465 ComputeTimingTable *table = ctx->compute_timing_table; 1466 /* NOTE(rnp): do a little spin to let this finish updating */ 1467 spin_while_true(table->write_index != atomic_load_u32(&table->read_index)); 1468 ComputeShaderStats *stats = ctx->compute_shader_stats; 1469 1470 u64 size = beamformer_shared_memory_allocation_size(sm, ec->memory); 1471 if (sizeof(stats->table) <= size) 1472 memory_copy(beamformer_shared_memory_allocation_pointer(sm, ec->memory), &stats->table, sizeof(stats->table)); 1473 }break; 1474 InvalidDefaultCase; 1475 } 1476 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync); 1477 }break; 1478 1479 case BeamformerWorkKind_CreateFilter:{ 1480 BeamformerCreateFilterContext *fctx = &work->create_filter_context; 1481 u32 block = fctx->parameter_block; 1482 u32 slot = fctx->filter_slot; 1483 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena); 1484 cp->filter_parameters[slot] = fctx->parameters; 1485 }break; 1486 1487 case BeamformerWorkKind_WaitThenCompute: 1488 case BeamformerWorkKind_Compute: 1489 { 1490 push_compute_timing_info(ctx->compute_timing_table, 1491 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin}); 1492 1493 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_set, arena); 1494 if unlikely(beamformer_parameter_block_dirty(sm, work->compute_context.parameter_set)) { 1495 u32 set = work->compute_context.parameter_set; 1496 Temp scratch = temp_begin(arena); 1497 beamformer_commit_parameter_block(ctx, cp, set, arena); 1498 temp_end(scratch); 1499 } 1500 1501 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute); 1502 1503 u32 dirty_programs = atomic_swap_u32(&cp->dirty_programs, 0); 1504 static_assert(BeamformerMaxComputeShaderStages <= 32, ""); 1505 if unlikely(dirty_programs) { 1506 for EachBit(dirty_programs, slot) { 1507 assert(slot < BeamformerMaxComputeShaderStages); 1508 Temp scratch; 1509 DeferLoop(scratch = temp_begin(arena), temp_end(scratch)) 1510 { 1511 u64 gpu_heap_pointer = cp->gpu_temp_arena.gpu_pointer; 1512 i32 index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1513 BeamformerShaderReloadInfo info = { 1514 .shader = cp->pipeline.shaders[slot], 1515 .shader_kind = VulkanShaderKind_Compute, 1516 .shader_descriptor = cp->shader_descriptors + slot, 1517 .gpu_heap_pointer = cp->shader_descriptors[slot].uses_heap ? gpu_heap_pointer : 0, 1518 .filename_or_data = BakeShaders ? beamformer_shader_data[index][0] 1519 : beamformer_reloadable_shader_files[index][0], 1520 .layout = cp->shader_descriptors[slot].layout, 1521 }; 1522 beamformer_reload_pipeline(cp->vulkan_pipelines + slot, &info, 1, arena); 1523 } 1524 } 1525 } 1526 1527 atomic_store_u32(&cs->processing_compute, 1); 1528 1529 start_renderdoc_capture(); 1530 1531 i32 das_index = -1; 1532 i32 coherency_weighting = -1; 1533 for (u32 i = 0; i < cp->pipeline.shader_count; i++) { 1534 if (cp->pipeline.shaders[i] == BeamformerShaderKind_CoherencyWeighting) 1535 coherency_weighting = (i32)i; 1536 1537 if (cp->pipeline.shaders[i] == BeamformerShaderKind_DAS) 1538 das_index = (i32)i; 1539 } 1540 1541 BeamformerFrame *frame = beamformer_frame_next(cs, cp->output_points, cp->iq_pipeline); 1542 frame->acquisition_kind = cp->acquisition_kind; 1543 frame->contrast_mode = cp->contrast_mode; 1544 frame->compound_count = cp->acquisition_count; 1545 frame->parameter_block = work->compute_context.parameter_set; 1546 frame->view_plane_tag = work->compute_context.view_plane; 1547 frame->focus_depth = work->compute_context.focus_depth; 1548 frame->steering_angle = work->compute_context.steering_angle; 1549 frame->transmit_receive_orientation = work->compute_context.tx_rx_orientation; 1550 1551 b32 transform_changed = !m4_equal(cp->last_voxel_transform, work->compute_context.voxel_transform); 1552 memory_copy(&cp->last_voxel_transform, &work->compute_context.voxel_transform, sizeof(cp->last_voxel_transform)); 1553 memory_copy(&frame->voxel_transform, &cp->last_voxel_transform, sizeof(frame->voxel_transform)); 1554 1555 if (transform_changed) { 1556 atomic_store_u32(&beamformer_context->ui_dirty_parameter_blocks, 1u << work->compute_context.parameter_set); 1557 cp->ui_voxel_transform = m4_identity(); 1558 } else if (u64_hash_from_str8(str8_struct(&cp->last_voxel_transform)) == cp->ui_last_voxel_transform_expected_hash) { 1559 frame->voxel_transform = m4_mul(cp->ui_voxel_transform, frame->voxel_transform); 1560 } 1561 1562 GPUCommandList cmd = gpu_command_list_begin(GPUTimeline_Compute); 1563 gpu_command_timestamp(cmd); 1564 1565 if (das_index >= 0) { 1566 GPUBuffer *backlog = cs->backlog.buffer; 1567 u64 frame_size = beamformer_frame_byte_size(frame->points, frame->data_kind); 1568 u64 offset = frame->gpu_pointer - backlog->gpu_pointer; 1569 gpu_command_clear_buffer(cmd, backlog, offset, frame_size, 0); 1570 } 1571 1572 if (coherency_weighting >= 0) { 1573 BeamformerCoherencyWeightingBakeParameters *cw = &cp->shader_descriptors[coherency_weighting].bake.CoherencyWeighting; 1574 GPUBuffer *gpu_arena = &cp->gpu_temp_arena; 1575 u64 coherent_size = beamformer_incoherent_frame_byte_size(frame->points, frame->data_kind); 1576 gpu_command_clear_buffer(cmd, gpu_arena, cw->IncoherentSum, coherent_size, 0); 1577 } 1578 1579 BeamformerRFBuffer *rf = &cs->rf_buffer; 1580 u64 compute_index = rf->compute_index; 1581 u64 slot = compute_index % countof(rf->upload_complete_values); 1582 1583 // NOTE(rnp): the library/ui will ensure that any time a new dataset is uploaded 1584 // the next time beamforming is issued it will be tagged as WaitThenCompute. 1585 if (work->kind != BeamformerWorkKind_WaitThenCompute) 1586 slot = (compute_index - 1) % countof(rf->upload_complete_values); 1587 1588 // NOTE(rnp): nvidia needs a memory barrier between pipeline stages 1589 // for correct output. It doesn't seem to effect performance on nvidia cards. 1590 // It does have a very minor performance decrease on AMD, but technically its 1591 // against vulkan spec to skip them so for simplicity we will default to for now 1592 b32 pipeline_memory_barrier = 1; //gpu_info()->vendor == GPUVendor_NVIDIA; 1593 1594 for (u32 channel_offset = 0; 1595 channel_offset < cp->channel_count; 1596 channel_offset += BeamformerChunkChannelCount) 1597 { 1598 u64 rf_pointer = rf->buffer.gpu_pointer + slot * rf->active_rf_size; 1599 rf_pointer += cp->raw_channel_byte_stride * channel_offset; 1600 1601 // NOTE(rnp): handle the case where the channel count is not a multiple of 1602 // BeamformerChunkChannelCount. We do not want to have to modify every shader 1603 // that might run to care about this so here we run a copy/clear to pad the 1604 // data for the last iteration. 1605 b32 special_handling = (cp->channel_count - channel_offset) < BeamformerChunkChannelCount; 1606 if unlikely(special_handling) { 1607 u32 channel_pad_count = BeamformerChunkChannelCount - (cp->channel_count - channel_offset); 1608 u32 input_index = cs->ping_pong_input_index; 1609 u64 pp_size = cs->ping_pong_buffer.size / PING_PONG_BUFFER_SLOTS; 1610 u64 copy_size = cp->raw_channel_byte_stride * (cp->channel_count - channel_offset); 1611 u64 copy_offset = rf_pointer - rf->buffer.gpu_pointer; 1612 u64 clear_size = cp->raw_channel_byte_stride * channel_pad_count; 1613 1614 gpu_command_copy_buffer(cmd, &cs->ping_pong_buffer, input_index * pp_size, &rf->buffer, copy_offset, copy_size); 1615 gpu_command_clear_buffer(cmd, &cs->ping_pong_buffer, input_index * pp_size + copy_size, clear_size, 0); 1616 // TODO(rnp): technically this should be a transfer to compute barrier 1617 // but a full memory barrier will also do the trick 1618 gpu_command_pipeline_barrier(cmd, 1); 1619 } 1620 1621 for (u32 i = 0; i < cp->first_image_shader_index; i++) { 1622 u32 output_index = !cs->ping_pong_input_index; 1623 u32 input_index = cs->ping_pong_input_index; 1624 1625 u64 pp_size = cs->ping_pong_buffer.size / PING_PONG_BUFFER_SLOTS; 1626 u64 pp_input_pointer = cs->ping_pong_buffer.gpu_pointer + input_index * pp_size; 1627 u64 pp_output_pointer = cs->ping_pong_buffer.gpu_pointer + output_index * pp_size; 1628 1629 u64 output_pointer = pp_output_pointer; 1630 u64 input_pointer = (i == 0 && !special_handling) ? rf_pointer : pp_input_pointer; 1631 1632 if (i != 0) gpu_command_pipeline_barrier(cmd, pipeline_memory_barrier); 1633 do_compute_shader(cmd, cp, frame, input_pointer, output_pointer, i, channel_offset); 1634 gpu_command_timestamp(cmd); 1635 } 1636 } 1637 1638 for (u32 i = cp->first_image_shader_index; i < cp->pipeline.shader_count; i++) { 1639 gpu_command_pipeline_barrier(cmd, pipeline_memory_barrier); 1640 do_compute_shader(cmd, cp, frame, 0, 0, i, 0); 1641 gpu_command_timestamp(cmd); 1642 } 1643 1644 u32 wait_info_count = 0; 1645 GPUSemaphoreSignalInfo wait_info = {.semaphore = rf->upload_semaphore}; 1646 if (work->kind == BeamformerWorkKind_WaitThenCompute) { 1647 spin_while_true(atomic_load_u64(&rf->insertion_index) <= compute_index); 1648 wait_info.value = atomic_load_u64(rf->upload_complete_values + slot); 1649 wait_info_count = 1; 1650 } 1651 1652 u64 end_timeline_value = gpu_command_list_end(cmd, &wait_info, wait_info_count, 0, 0); 1653 atomic_store_u64(rf->compute_complete_values + slot, end_timeline_value); 1654 1655 if (work->kind == BeamformerWorkKind_WaitThenCompute) 1656 atomic_add_u64(&rf->compute_index, 1); 1657 1658 atomic_store_u64(&frame->timeline_valid_value, end_timeline_value); 1659 1660 Temp scratch; 1661 DeferLoop(scratch = temp_begin(arena), temp_end(scratch)) 1662 { 1663 /* NOTE(rnp): this blocks until work completes */ 1664 u64 count = 0; 1665 u64 *timestamps = gpu_read_timestamps(GPUTimeline_Compute, &count, arena); 1666 1667 i32 steps = ((i32)cp->channel_count / BeamformerChunkChannelCount) - 1; 1668 i32 step = 0; 1669 u32 shader_index = 0; 1670 u64 last_time = count > 0 ? timestamps[0] : 0; 1671 1672 for (u64 i = 1; i < count; i++) { 1673 push_compute_timing_info(ctx->compute_timing_table, (ComputeTimingInfo){ 1674 .kind = ComputeTimingInfoKind_Shader, 1675 .shader = cp->pipeline.shaders[shader_index], 1676 .shader_slot = shader_index, 1677 .timer_count = timestamps[i] - last_time, 1678 }); 1679 last_time = timestamps[i]; 1680 1681 shader_index++; 1682 if (shader_index == cp->first_image_shader_index && step < steps) { 1683 shader_index = 0; 1684 step++; 1685 } 1686 } 1687 } 1688 1689 cs->processing_progress = 1; 1690 1691 //if (has_sum) { 1692 if (0) { 1693 #if 0 1694 u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames)); 1695 ctx->averaged_frames[aframe_index].view_plane_tag = frame->view_plane_tag; 1696 ctx->averaged_frames[aframe_index].ready_to_present = 1; 1697 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index)); 1698 #endif 1699 } else { 1700 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame); 1701 } 1702 1703 atomic_store_u32(&cs->processing_compute, 0); 1704 1705 push_compute_timing_info(ctx->compute_timing_table, 1706 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd}); 1707 1708 end_renderdoc_capture(); 1709 }break; 1710 InvalidDefaultCase; 1711 } 1712 } 1713 } 1714 1715 function void 1716 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats) 1717 { 1718 /* TODO(rnp): we do not currently do anything to handle the potential for a half written 1719 * info item. this could result in garbage entries but they shouldn't really matter */ 1720 1721 u32 target = atomic_load_u32(&t->write_index); 1722 u32 stats_index = stats->latest_frame_index; 1723 1724 b32 has_rf = 0; 1725 f32 gpu_clocks_to_nano = 1.0e-9f * gpu_info()->timestamp_period_ns; 1726 1727 // NOTE(rnp): not equal (the index may wrap) 1728 while (t->read_index != target) { 1729 ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)]; 1730 switch (info.kind) { 1731 1732 case ComputeTimingInfoKind_ComputeFrameBegin:{ 1733 assert(t->compute_frame_active == 0); 1734 t->compute_frame_active = 1; 1735 /* NOTE(rnp): allow multiple instances of same shader to accumulate */ 1736 t->in_flight_shader_count = 0; 1737 memory_clear(t->in_flight_shader_ids, 0, sizeof(t->in_flight_shader_ids)); 1738 memory_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index])); 1739 }break; 1740 1741 case ComputeTimingInfoKind_ComputeFrameEnd:{ 1742 assert(t->compute_frame_active == 1); 1743 t->compute_frame_active = 0; 1744 stats_index = stats->latest_frame_index = (stats_index + 1) % countof(stats->table.times); 1745 stats->table.shader_count = t->in_flight_shader_count; 1746 memory_copy(stats->table.shader_ids, t->in_flight_shader_ids, sizeof(t->in_flight_shader_ids)); 1747 }break; 1748 1749 case ComputeTimingInfoKind_Shader:{ 1750 t->in_flight_shader_count = Max(t->in_flight_shader_count, info.shader_slot + 1u); 1751 t->in_flight_shader_ids[info.shader_slot] = info.shader; 1752 stats->table.times[stats_index][info.shader_slot] += info.timer_count * gpu_clocks_to_nano; 1753 }break; 1754 1755 case ComputeTimingInfoKind_RF_Data:{ 1756 stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas); 1757 f32 delta = info.timer_count / (f32)os_system_info()->timer_frequency; 1758 stats->table.rf_time_deltas[stats->latest_rf_index] = delta; 1759 has_rf = 1; 1760 }break; 1761 } 1762 /* NOTE(rnp): do this at the end so that stats table is always in a consistent state */ 1763 t->read_index++; 1764 } 1765 1766 for (u32 i = 0; i < stats->table.shader_count; i++) { 1767 f32 sum = 0; 1768 for EachElement(stats->table.times, it) 1769 sum += stats->table.times[it][i]; 1770 stats->average_times[i] = sum / countof(stats->table.times); 1771 } 1772 1773 if (has_rf) { 1774 f32 sum = 0; 1775 for EachElement(stats->table.rf_time_deltas, i) 1776 sum += stats->table.rf_time_deltas[i]; 1777 stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas); 1778 } 1779 } 1780 1781 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute) 1782 { 1783 BeamformerSharedMemory *sm = ctx->shared_memory; 1784 complete_queue(ctx, &sm->external_work_queue, arena); 1785 complete_queue(ctx, ctx->beamform_work_queue, arena); 1786 } 1787 1788 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload) 1789 { 1790 BeamformerSharedMemory *sm = ctx->shared_memory; 1791 BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF; 1792 1793 BeamformerSharedMemoryAllocationHandle rf_handle; 1794 if (atomic_load_u32(sm->locks + upload_lock) && (rf_handle = atomic_swap_u64(&sm->rf_handle, 0))) 1795 { 1796 BeamformerRFBuffer *rf = ctx->rf_buffer; 1797 1798 u64 rf_size = beamformer_shared_memory_allocation_size(sm, rf_handle); 1799 1800 rf->active_rf_size = gpu_round_up_to_sync_size(rf_size, BEAMFORMER_RF_ALIGN); 1801 if unlikely((u64)rf->buffer.size < countof(rf->upload_complete_values) * rf->active_rf_size) { 1802 gpu_buffer_allocate(&rf->buffer, (GPUBufferAllocateInfo){ 1803 .size = countof(rf->upload_complete_values) * rf->active_rf_size, 1804 .flags = GPUUsageFlag_HostWrite, 1805 .label = str8("RawRFBuffer"), 1806 .timelines_used = (GPUTimeline []){GPUTimeline_Compute}, 1807 .timeline_count = 1, 1808 }); 1809 } 1810 1811 u64 slot = rf->insertion_index % countof(rf->upload_complete_values); 1812 1813 /* NOTE(rnp): don't overwrite slot if the compute thread hasn't processed it */ 1814 spin_while_true(rf->insertion_index - atomic_load_u64(&rf->compute_index) >= countof(rf->upload_complete_values)); 1815 gpu_host_wait_timeline(GPUTimeline_Compute, rf->compute_complete_values[slot], -1ULL); 1816 1817 assert((ctx->shared_memory_size % os_system_info()->page_size) == 0 && 1818 (os_system_info()->page_size % gpu_round_up_to_sync_size(1, BEAMFORMER_RF_ALIGN)) == 0); 1819 1820 u64 wait_value = gpu_semaphore_value(rf->upload_semaphore) + 1; 1821 atomic_store_u64(rf->upload_complete_values + slot, wait_value); 1822 atomic_add_u64(&rf->insertion_index, 1); 1823 1824 void *host_pointer = gpu_buffer_host_pointer(&rf->buffer); 1825 memory_copy_non_temporal((u8 *)host_pointer + slot * rf->active_rf_size, 1826 beamformer_shared_memory_allocation_pointer(sm, rf_handle), 1827 rf->active_rf_size); 1828 store_fence(); 1829 1830 GPUSemaphoreSignalInfo signal_info = {.semaphore = rf->upload_semaphore, .value = wait_value}; 1831 gpu_buffer_make_visible(&rf->buffer, slot * rf->active_rf_size, rf->active_rf_size, &signal_info, 1); 1832 1833 beamformer_shared_memory_allocation_release(sm, rf_handle); 1834 1835 post_sync_barrier(ctx->shared_memory, upload_lock); 1836 os_wake_all_waiters(ctx->compute_worker_sync); 1837 1838 u64 current_time = os_timer_count(); 1839 push_compute_timing_info(ctx->compute_timing_table, (ComputeTimingInfo){ 1840 .kind = ComputeTimingInfoKind_RF_Data, 1841 .timer_count = current_time - rf->timestamp, 1842 }); 1843 rf->timestamp = current_time; 1844 } 1845 } 1846 1847 function void 1848 beamformer_queue_compute(BeamformerCtx *ctx, BeamformerComputeWorkContext cw) 1849 { 1850 BeamformerSharedMemory *sm = ctx->shared_memory; 1851 BeamformerSharedMemoryLockKind dispatch_lock = BeamformerSharedMemoryLockKind_DispatchCompute; 1852 if (!sm->live_imaging_parameters.active && beamformer_shared_memory_take_lock(sm, (i32)dispatch_lock, 0)) 1853 { 1854 BeamformWork *work = beamform_work_queue_push(ctx->beamform_work_queue); 1855 if (work) { 1856 work->kind = BeamformerWorkKind_Compute; 1857 memory_copy(&work->compute_context, &cw, sizeof(cw)); 1858 beamform_work_queue_push_commit(ctx->beamform_work_queue); 1859 } 1860 } 1861 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1862 } 1863 1864 #include "ui.c" 1865 1866 function void 1867 beamformer_process_input_events(BeamformerCtx *ctx, BeamformerInput *input, 1868 BeamformerInputEvent *events, u32 event_count) 1869 { 1870 for (u32 index = 0; index < event_count; index++) { 1871 BeamformerInputEvent *event = events + index; 1872 switch (event->kind) { 1873 1874 // NOTE(rnp): ui will handle these 1875 case BeamformerInputEventKind_ButtonPress: 1876 case BeamformerInputEventKind_ButtonRelease: 1877 case BeamformerInputEventKind_MouseScroll: 1878 case BeamformerInputEventKind_WindowResize: 1879 {}break; 1880 1881 case BeamformerInputEventKind_ExecutableReload:{ 1882 ui_init(ctx, ctx->ui_arena); 1883 }break; 1884 1885 case BeamformerInputEventKind_FileEvent:{ 1886 BeamformerFileReloadContext *frc = event->file_watch_user_context; 1887 switch (frc->kind) { 1888 case BeamformerFileReloadKind_ComputeShader:{ 1889 for EachElement(ctx->compute_context.compute_plans, block) { 1890 BeamformerComputePlan *cp = ctx->compute_context.compute_plans[block]; 1891 for (u32 slot = 0; cp && slot < cp->pipeline.shader_count; slot++) { 1892 i32 shader_index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1893 if (beamformer_reloadable_shader_kinds[shader_index] == frc->shader_reload.shader) 1894 atomic_or_u32(&cp->dirty_programs, 1 << slot); 1895 } 1896 } 1897 1898 // TODO(rnp): track latest parameter block 1899 if (ctx->latest_frame) { 1900 BeamformerFrame *f = ctx->latest_frame; 1901 beamformer_queue_compute(ctx, (BeamformerComputeWorkContext){ 1902 .voxel_transform = f->voxel_transform, 1903 .rf_offset = 0, 1904 .focus_depth = f->focus_depth, 1905 .steering_angle = f->steering_angle, 1906 .tx_rx_orientation = f->transmit_receive_orientation, 1907 .parameter_set = 0, 1908 .view_plane = f->view_plane_tag, 1909 }); 1910 } 1911 }break; 1912 1913 case BeamformerFileReloadKind_RenderShader:{ 1914 beamformer_reload_render_pipeline(frc->shader_reload.pipeline, frc->shader_reload.shader, ctx->arena); 1915 ctx->render_shader_updated = 1; 1916 }break; 1917 1918 InvalidDefaultCase; 1919 } 1920 }break; 1921 1922 InvalidDefaultCase; 1923 } 1924 } 1925 } 1926 1927 function void 1928 beamformer_panel_group_insert_at(BeamformerUIPanel *group, BeamformerUIPanel *tab, u64 new_child_index) 1929 { 1930 if (tab->parent) beamformer_ui_panel_unlink(tab); 1931 new_child_index = Min(new_child_index, group->child_count); 1932 1933 tab->parent = group; 1934 group->child_count++; 1935 if (group->kind == BeamformerPanelKind_TabGroup) group->u.tab_focus = tab; 1936 1937 BeamformerUIPanel *previous_sibling = new_child_index == 0 ? 0 : group->first_child; 1938 for (u64 child_index = 1; child_index < new_child_index; child_index++) 1939 previous_sibling = previous_sibling->next_sibling; 1940 1941 if (previous_sibling) { 1942 tab->previous_sibling = previous_sibling; 1943 tab->next_sibling = previous_sibling->next_sibling; 1944 if (tab->next_sibling) tab->next_sibling->previous_sibling = tab; 1945 previous_sibling->next_sibling = tab; 1946 if (previous_sibling == group->last_child) group->last_child = tab; 1947 } else { 1948 DLLInsertFirst(0, group->first_child, group->last_child, tab, next_sibling, previous_sibling); 1949 } 1950 } 1951 1952 BEAMFORMER_EXPORT void 1953 beamformer_frame_step(void *memory, BeamformerInput *input) 1954 { 1955 BeamformerCtx *ctx = beamformer_context = memory; 1956 beamformer_input = input; 1957 1958 u64 current_time = os_timer_count(); 1959 dt_for_frame = (f64)(current_time - ctx->frame_timestamp) / os_system_info()->timer_frequency; 1960 ctx->frame_timestamp = current_time; 1961 ctx->frame_index++; 1962 1963 coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats); 1964 1965 // NOTE(rnp): reset frame state 1966 { 1967 ctx->registers = &ctx->base_registers; 1968 swap(ctx->command_queues[0], ctx->command_queues[1]); 1969 zero_struct(ctx->command_queues + 0); 1970 //zero_struct(ctx->registers); 1971 arena_clear(beamformer_frame_arena()); 1972 } 1973 1974 beamformer_process_input_events(ctx, input, input->event_queue, input->event_count); 1975 1976 BeamformerSharedMemory *sm = ctx->shared_memory; 1977 u32 live_imaging_active = atomic_load_u32(&sm->live_imaging_parameters.active); 1978 if (live_imaging_active != ctx->live_imaging_active) { 1979 if (ctx->live_imaging_active) { 1980 if (ctx->auto_live_control_panel) { 1981 BeamformerUIPanel *parent = ctx->auto_live_control_panel->parent; 1982 beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)ctx->auto_live_control_panel); 1983 if (parent->child_count == 1) 1984 beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)parent); 1985 } 1986 } else { 1987 if (beamformer_registers()->live_controls) { 1988 beamformer_command(beamformer_command_infos[BeamformerCommandKind_FocusTab].string, 1989 .tree_node = beamformer_registers()->live_controls); 1990 } else { 1991 ctx->auto_live_control_panel = beamformer_ui_push_panel(0, BeamformerPanelKind_LiveImagingControls); 1992 beamformer_command(beamformer_command_infos[BeamformerCommandKind_SplitTree].string, 1993 .tree_node = (u64)ctx->auto_live_control_panel, 1994 .split_axis = Axis2_X, 1995 .split_left_tree = (u64)ui_context->tree, 1996 .split_right_tree = 0, 1997 .drop_target_tree = (u64)ui_context->tree); 1998 } 1999 ctx->live_imaging_active_frame = ctx->frame_index; 2000 } 2001 ctx->live_imaging_active = live_imaging_active; 2002 } 2003 2004 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_UploadRF)) 2005 os_wake_all_waiters(&ctx->upload_worker.sync_variable); 2006 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_DispatchCompute)) 2007 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 2008 2009 beamformer_registers()->frame = (u64)(ctx->latest_frame - ctx->compute_context.backlog.frames); 2010 2011 beamformer_ui_frame(); 2012 2013 // NOTE(rnp): execute commands 2014 for (BeamformerCommandNode *node = ctx->command_queues[0].first; 2015 node; 2016 node = node == node->next ? 0 : node->next) 2017 { 2018 BeamformerRegistersScope() 2019 { 2020 memory_copy(beamformer_registers(), node->command.registers, sizeof(*node->command.registers)); 2021 BeamformerCommandKind kind = beamformer_command_kind_from_string(node->command.name); 2022 switch (kind) { 2023 InvalidDefaultCase; 2024 case BeamformerCommandKind_CloseTab:{ 2025 BeamformerUIPanel *tab = (BeamformerUIPanel *)beamformer_registers()->tree_node; 2026 ui_kill_panel(tab); 2027 }break; 2028 2029 case BeamformerCommandKind_FocusTab:{ 2030 BeamformerUIPanel *tab = (BeamformerUIPanel *)beamformer_registers()->tree_node; 2031 assert(tab->parent->kind == BeamformerPanelKind_TabGroup); 2032 tab->parent->u.tab_focus = tab; 2033 }break; 2034 2035 case BeamformerCommandKind_MoveTab:{ 2036 BeamformerUIPanel *move = (BeamformerUIPanel *)beamformer_registers()->tree_node; 2037 BeamformerUIPanel *group = (BeamformerUIPanel *)beamformer_registers()->drop_target_tree; 2038 BeamformerUIPanel *parent = move->parent; 2039 u64 new_child_index = beamformer_registers()->drop_child_index; 2040 beamformer_panel_group_insert_at(group, move, new_child_index); 2041 2042 if (move->kind == BeamformerPanelKind_LiveImagingControls) { 2043 beamformer_context->base_registers.v.live_controls = (u64)move; 2044 if (move == ctx->auto_live_control_panel) 2045 ctx->auto_live_control_panel = 0; 2046 } 2047 2048 if (parent->child_count == 0) 2049 beamformer_command(beamformer_command_infos[BeamformerCommandKind_CloseTab].string, .tree_node = (u64)parent); 2050 }break; 2051 2052 case BeamformerCommandKind_OpenTab:{ 2053 BeamformerUIPanel *panel = (BeamformerUIPanel *)beamformer_registers()->tree_node; 2054 assert(panel->kind == BeamformerPanelKind_TabGroup); 2055 2056 BeamformerPanelKind new_panel_kind = beamformer_panel_kind_from_string(beamformer_registers()->string); 2057 beamformer_ui_push_panel(panel, new_panel_kind); 2058 }break; 2059 2060 case BeamformerCommandKind_SplitTree:{ 2061 BeamformerUIPanel *drag = (BeamformerUIPanel *)beamformer_registers()->tree_node; 2062 BeamformerUIPanel *left = (BeamformerUIPanel *)beamformer_registers()->split_left_tree; 2063 BeamformerUIPanel *right = (BeamformerUIPanel *)beamformer_registers()->split_right_tree; 2064 Axis2 axis = beamformer_registers()->split_axis; 2065 2066 BeamformerUIPanel *new_split = beamformer_ui_push_panel(0, BeamformerPanelKind_Split); 2067 BeamformerUIPanel *new_tab_group = beamformer_ui_push_panel(0, BeamformerPanelKind_TabGroup); 2068 beamformer_panel_group_insert_at(new_tab_group, drag, 0); 2069 2070 BeamformerUIPanel *target = 0; 2071 u32 target_child_index = 0; 2072 f32 new_split_pct = 0.5f; 2073 2074 if (left == 0 || right == 0) { 2075 // NOTE(rnp): split on edge of window 2076 target = left ? left : right; 2077 target_child_index = left ? 0 : 1; 2078 2079 if (target->kind == BeamformerPanelKind_TabGroup) { 2080 new_split->kind = BeamformerPanelKind_TabGroup; 2081 new_split->u.tab_focus = target->u.tab_focus; 2082 } 2083 2084 for (BeamformerUIPanel *child = target->last_child, *next; child; child = next) { 2085 next = child->previous_sibling; 2086 beamformer_panel_group_insert_at(new_split, child, 0); 2087 } 2088 2089 beamformer_panel_group_insert_at(target, new_tab_group, 0); 2090 } else if (((drag == left) && right->kind == BeamformerPanelKind_Split) || 2091 ((drag == right) && left->kind == BeamformerPanelKind_Split)) 2092 { 2093 // NOTE(rnp): split on internal split 2094 target = left == drag ? right : left; 2095 target_child_index = 1; 2096 new_split_pct = 1.f / 3.f; 2097 beamformer_panel_group_insert_at(new_split, new_tab_group, 0); 2098 beamformer_panel_group_insert_at(new_split, target->last_child, 1); 2099 } else { 2100 // NOTE(rnp): TabGroup Split 2101 target = left == drag ? right : left; 2102 target_child_index = left == drag ? 1 : 0; 2103 assert(target->kind == BeamformerPanelKind_TabGroup); 2104 2105 BeamformerUIPanel *focus = target->u.tab_focus; 2106 new_split->kind = BeamformerPanelKind_TabGroup; 2107 for (BeamformerUIPanel *child = target->last_child, *next; child; child = next) { 2108 next = child->previous_sibling; 2109 beamformer_panel_group_insert_at(new_split, child, 0); 2110 } 2111 new_split->u.tab_focus = focus; 2112 2113 beamformer_panel_group_insert_at(target, new_tab_group, 0); 2114 } 2115 2116 beamformer_panel_group_insert_at(target, new_split, target_child_index); 2117 if (target->kind == BeamformerPanelKind_Split) { 2118 new_split->u.split.axis = target->u.split.axis; 2119 new_split->u.split.fraction = target->u.split.fraction; 2120 } 2121 target->kind = BeamformerPanelKind_Split; 2122 target->u.split.axis = axis; 2123 target->u.split.fraction = new_split_pct; 2124 }break; 2125 2126 } 2127 } 2128 } 2129 2130 ctx->render_shader_updated = 0; 2131 }