beamformer_core.c (59665B)
1 /* See LICENSE for license details. */ 2 /* TODO(rnp): 3 * [ ]: refactor: DecodeMode_None should use a different mapping and optional conversion shader 4 * for rf only mode with no filter and demod/filter should gain the OutputFloats flag for iq 5 * case and rf mode with filter; this can also be used instead of first pass uniform 6 * [ ]: refactor: replace UploadRF with just the scratch_rf_size variable, 7 * use below to spin wait in library 8 * [ ]: utilize umonitor/umwait (intel), monitorx/mwaitx (amd), and wfe/sev (aarch64) 9 * for power efficient low latency waiting 10 * [ ]: refactor: split decode into reshape and decode 11 * - the check for first pass reshaping is the last non constant check 12 * in the shader 13 * - this will also remove the need for the channel mapping in the decode shader 14 * [X]: refactor: ui: reload only shader which is affected by the interaction 15 * [ ]: BeamformWorkQueue -> BeamformerWorkQueue 16 * [ ]: need to keep track of gpu memory in some way 17 * - want to be able to store more than 16 2D frames but limit 3D frames 18 * - maybe keep track of how much gpu memory is committed for beamformed images 19 * and use that to determine when to loop back over existing textures 20 * - to do this maybe use a circular linked list instead of a flat array 21 * - then have a way of querying how many frames are available for a specific point count 22 * [ ]: bug: reinit cuda on hot-reload 23 */ 24 25 #include "compiler.h" 26 27 #if defined(BEAMFORMER_DEBUG) && !defined(BEAMFORMER_EXPORT) && OS_WINDOWS 28 #define BEAMFORMER_EXPORT __declspec(dllexport) 29 #endif 30 31 #include "beamformer_internal.h" 32 33 global f32 dt_for_frame; 34 35 #define DECODE_FIRST_PASS_UNIFORM_LOC 1 36 37 #define DAS_CYCLE_T_UNIFORM_LOC 2 38 #define DAS_FAST_CHANNEL_UNIFORM_LOC 3 39 40 #define MIN_MAX_MIPS_LEVEL_UNIFORM_LOC 1 41 #define SUM_PRESCALE_UNIFORM_LOC 1 42 43 #if !BEAMFORMER_RENDERDOC_HOOKS 44 #define start_renderdoc_capture(...) 45 #define end_renderdoc_capture(...) 46 #define renderdoc_attached(...) (0) 47 #else 48 global renderdoc_start_frame_capture_fn *start_frame_capture; 49 global renderdoc_end_frame_capture_fn *end_frame_capture; 50 #define start_renderdoc_capture(gl) if (start_frame_capture) start_frame_capture(gl, 0) 51 #define end_renderdoc_capture(gl) if (end_frame_capture) end_frame_capture(gl, 0) 52 #define renderdoc_attached(...) (start_frame_capture != 0) 53 #endif 54 55 typedef struct { 56 BeamformerFrame *frames; 57 u32 capacity; 58 u32 offset; 59 u32 cursor; 60 u32 needed_frames; 61 } ComputeFrameIterator; 62 63 function void 64 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block) 65 { 66 assert(block < countof(cc->compute_plans)); 67 BeamformerComputePlan *cp = cc->compute_plans[block]; 68 if (cp) { 69 glDeleteBuffers(countof(cp->ubos), cp->ubos); 70 glDeleteTextures(countof(cp->textures), cp->textures); 71 for (u32 i = 0; i < countof(cp->filters); i++) 72 glDeleteBuffers(1, &cp->filters[i].ssbo); 73 cc->compute_plans[block] = 0; 74 SLLPushFreelist(cp, cc->compute_plan_freelist); 75 } 76 } 77 78 function BeamformerComputePlan * 79 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena) 80 { 81 assert(block < countof(cc->compute_plans)); 82 BeamformerComputePlan *result = cc->compute_plans[block]; 83 if (!result) { 84 result = SLLPopFreelist(cc->compute_plan_freelist); 85 if (!result) result = push_struct_no_zero(arena, BeamformerComputePlan); 86 zero_struct(result); 87 cc->compute_plans[block] = result; 88 89 result->ui_voxel_transform = m4_identity(); 90 91 glCreateBuffers(countof(result->ubos), result->ubos); 92 93 Stream label = arena_stream(*arena); 94 #define X(k, t, ...) \ 95 glNamedBufferStorage(result->ubos[BeamformerComputeUBOKind_##k], sizeof(t), \ 96 0, GL_DYNAMIC_STORAGE_BIT); \ 97 stream_append_s8(&label, s8(#t "[")); \ 98 stream_append_u64(&label, block); \ 99 stream_append_byte(&label, ']'); \ 100 glObjectLabel(GL_BUFFER, result->ubos[BeamformerComputeUBOKind_##k], \ 101 label.widx, (c8 *)label.data); \ 102 label.widx = 0; 103 BEAMFORMER_COMPUTE_UBO_LIST 104 #undef X 105 106 #define X(_k, t, ...) t, 107 GLenum gl_kind[] = {BEAMFORMER_COMPUTE_TEXTURE_LIST_FULL}; 108 #undef X 109 read_only local_persist s8 tex_prefix[] = { 110 #define X(k, ...) s8_comp(#k "["), 111 BEAMFORMER_COMPUTE_TEXTURE_LIST_FULL 112 #undef X 113 }; 114 glCreateTextures(GL_TEXTURE_1D, BeamformerComputeTextureKind_Count - 1, result->textures); 115 for (u32 i = 0; i < BeamformerComputeTextureKind_Count - 1; i++) { 116 /* TODO(rnp): this could be predicated on channel count for this compute plan */ 117 glTextureStorage1D(result->textures[i], 1, gl_kind[i], BeamformerMaxChannelCount); 118 stream_append_s8(&label, tex_prefix[i]); 119 stream_append_u64(&label, block); 120 stream_append_byte(&label, ']'); 121 glObjectLabel(GL_TEXTURE, result->textures[i], label.widx, (c8 *)label.data); 122 label.widx = 0; 123 } 124 } 125 return result; 126 } 127 128 function void 129 beamformer_filter_update(BeamformerFilter *f, BeamformerFilterParameters fp, u32 block, u32 slot, Arena arena) 130 { 131 Stream sb = arena_stream(arena); 132 stream_append_s8s(&sb, 133 beamformer_filter_kind_strings[fp.kind % countof(beamformer_filter_kind_strings)], 134 s8("Filter[")); 135 stream_append_u64(&sb, block); 136 stream_append_s8(&sb, s8("][")); 137 stream_append_u64(&sb, slot); 138 stream_append_byte(&sb, ']'); 139 s8 label = arena_stream_commit(&arena, &sb); 140 141 void *filter = 0; 142 switch (fp.kind) { 143 case BeamformerFilterKind_Kaiser:{ 144 /* TODO(rnp): this should also support complex */ 145 /* TODO(rnp): implement this as an IFIR filter instead to reduce computation */ 146 filter = kaiser_low_pass_filter(&arena, fp.kaiser.cutoff_frequency, fp.sampling_frequency, 147 fp.kaiser.beta, (i32)fp.kaiser.length); 148 f->length = (i32)fp.kaiser.length; 149 f->time_delay = (f32)f->length / 2.0f / fp.sampling_frequency; 150 }break; 151 case BeamformerFilterKind_MatchedChirp:{ 152 typeof(fp.matched_chirp) *mc = &fp.matched_chirp; 153 f32 fs = fp.sampling_frequency; 154 f->length = (i32)(mc->duration * fs); 155 if (fp.complex) { 156 filter = baseband_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1, 0.5f); 157 f->time_delay = complex_filter_first_moment(filter, f->length, fs); 158 } else { 159 filter = rf_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1); 160 f->time_delay = real_filter_first_moment(filter, f->length, fs); 161 } 162 }break; 163 InvalidDefaultCase; 164 } 165 166 f->parameters = fp; 167 168 glDeleteBuffers(1, &f->ssbo); 169 glCreateBuffers(1, &f->ssbo); 170 glNamedBufferStorage(f->ssbo, f->length * (i32)sizeof(f32) * (fp.complex? 2 : 1), filter, 0); 171 glObjectLabel(GL_BUFFER, f->ssbo, (i32)label.len, (c8 *)label.data); 172 } 173 174 function ComputeFrameIterator 175 compute_frame_iterator(BeamformerCtx *ctx, u32 start_index, u32 needed_frames) 176 { 177 start_index = start_index % ARRAY_COUNT(ctx->beamform_frames); 178 179 ComputeFrameIterator result; 180 result.frames = ctx->beamform_frames; 181 result.offset = start_index; 182 result.capacity = ARRAY_COUNT(ctx->beamform_frames); 183 result.cursor = 0; 184 result.needed_frames = needed_frames; 185 return result; 186 } 187 188 function BeamformerFrame * 189 frame_next(ComputeFrameIterator *bfi) 190 { 191 BeamformerFrame *result = 0; 192 if (bfi->cursor != bfi->needed_frames) { 193 u32 index = (bfi->offset + bfi->cursor++) % bfi->capacity; 194 result = bfi->frames + index; 195 } 196 return result; 197 } 198 199 function b32 200 beamformer_frame_compatible(BeamformerFrame *f, iv3 dim, GLenum gl_kind) 201 { 202 b32 result = gl_kind == f->gl_kind && iv3_equal(dim, f->dim); 203 return result; 204 } 205 206 function iv3 207 das_valid_points(iv3 points) 208 { 209 iv3 result; 210 result.x = Max(points.x, 1); 211 result.y = Max(points.y, 1); 212 result.z = Max(points.z, 1); 213 return result; 214 } 215 216 function void 217 alloc_beamform_frame(BeamformerFrame *out, iv3 out_dim, GLenum gl_kind, s8 name, Arena arena) 218 { 219 out->dim = das_valid_points(out_dim); 220 221 /* NOTE: allocate storage for beamformed output data; 222 * this is shared between compute and fragment shaders */ 223 u32 max_dim = (u32)Max(out->dim.x, Max(out->dim.y, out->dim.z)); 224 out->mips = (i32)ctz_u32(round_up_power_of_2(max_dim)) + 1; 225 226 out->gl_kind = gl_kind; 227 228 Stream label = arena_stream(arena); 229 stream_append_s8(&label, name); 230 stream_append_byte(&label, '['); 231 stream_append_hex_u64(&label, out->id); 232 stream_append_byte(&label, ']'); 233 234 glDeleteTextures(1, &out->texture); 235 glCreateTextures(GL_TEXTURE_3D, 1, &out->texture); 236 glTextureStorage3D(out->texture, out->mips, gl_kind, out->dim.x, out->dim.y, out->dim.z); 237 238 glTextureParameteri(out->texture, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 239 glTextureParameteri(out->texture, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 240 241 LABEL_GL_OBJECT(GL_TEXTURE, out->texture, stream_to_s8(&label)); 242 } 243 244 function void 245 update_hadamard_texture(BeamformerComputePlan *cp, i32 order, b32 row_major, Arena arena) 246 { 247 f16 *hadamard = make_hadamard_transpose(&arena, order, row_major); 248 if (hadamard) { 249 cp->hadamard_order = order; 250 u32 *texture = cp->textures + BeamformerComputeTextureKind_Hadamard; 251 glDeleteTextures(1, texture); 252 glCreateTextures(GL_TEXTURE_2D, 1, texture); 253 glTextureStorage2D(*texture, 1, GL_R16F, order, order); 254 glTextureSubImage2D(*texture, 0, 0, 0, order, order, GL_RED, GL_SHORT, hadamard); 255 256 Stream label = arena_stream(arena); 257 stream_append_s8(&label, s8("Hadamard")); 258 stream_append_i64(&label, order); 259 LABEL_GL_OBJECT(GL_TEXTURE, *texture, stream_to_s8(&label)); 260 } 261 } 262 263 function void 264 alloc_shader_storage(BeamformerCtx *ctx, u32 decoded_data_size, Arena arena) 265 { 266 BeamformerComputeContext *cc = &ctx->compute_context; 267 glDeleteBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos); 268 glCreateBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos); 269 270 cc->ping_pong_ssbo_size = decoded_data_size; 271 272 Stream label = arena_stream(arena); 273 stream_append_s8(&label, s8("PingPongSSBO[")); 274 i32 s_widx = label.widx; 275 for (i32 i = 0; i < countof(cc->ping_pong_ssbos); i++) { 276 glNamedBufferStorage(cc->ping_pong_ssbos[i], (iz)decoded_data_size, 0, 0); 277 stream_append_i64(&label, i); 278 stream_append_byte(&label, ']'); 279 LABEL_GL_OBJECT(GL_BUFFER, cc->ping_pong_ssbos[i], stream_to_s8(&label)); 280 stream_reset(&label, s_widx); 281 } 282 283 /* TODO(rnp): (25.08.04) cuda lib is heavily broken atm. First there are multiple RF 284 * buffers and cuda decode shouldn't assume that the data is coming from the rf_buffer 285 * ssbo. Second each parameter block may need a different hadamard matrix so ideally 286 * decode should just take the texture as a parameter. Third, none of these dimensions 287 * need to be pre-known by the library unless its allocating GPU memory which it shouldn't 288 * need to do. For now grab out of parameter block 0 but it is not correct */ 289 BeamformerParameterBlock *pb = beamformer_parameter_block(ctx->shared_memory, 0); 290 /* NOTE(rnp): these are stubs when CUDA isn't supported */ 291 cuda_register_buffers(cc->ping_pong_ssbos, countof(cc->ping_pong_ssbos), cc->rf_buffer.ssbo); 292 u32 decoded_data_dimension[3] = {pb->parameters.sample_count, pb->parameters.channel_count, pb->parameters.acquisition_count}; 293 cuda_init(pb->parameters.raw_data_dimensions.E, decoded_data_dimension); 294 } 295 296 function void 297 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info) 298 { 299 u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer); 300 t->buffer[index] = info; 301 } 302 303 function b32 304 fill_frame_compute_work(BeamformerCtx *ctx, BeamformWork *work, BeamformerViewPlaneTag plane, 305 u32 parameter_block, b32 indirect) 306 { 307 b32 result = work != 0; 308 if (result) { 309 u32 frame_id = atomic_add_u32(&ctx->next_render_frame_index, 1); 310 u32 frame_index = frame_id % countof(ctx->beamform_frames); 311 work->kind = indirect? BeamformerWorkKind_ComputeIndirect : BeamformerWorkKind_Compute; 312 work->lock = BeamformerSharedMemoryLockKind_DispatchCompute; 313 work->compute_context.parameter_block = parameter_block; 314 work->compute_context.frame = ctx->beamform_frames + frame_index; 315 work->compute_context.frame->ready_to_present = 0; 316 work->compute_context.frame->view_plane_tag = plane; 317 work->compute_context.frame->id = frame_id; 318 } 319 return result; 320 } 321 322 function void 323 do_sum_shader(BeamformerComputeContext *cc, u32 *in_textures, u32 in_texture_count, 324 u32 out_texture, iv3 out_data_dim) 325 { 326 /* NOTE: zero output before summing */ 327 glClearTexImage(out_texture, 0, GL_RED, GL_FLOAT, 0); 328 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 329 330 glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F); 331 for (u32 i = 0; i < in_texture_count; i++) { 332 glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 333 glDispatchCompute(ORONE((u32)out_data_dim.x / 32u), 334 ORONE((u32)out_data_dim.y), 335 ORONE((u32)out_data_dim.z / 32u)); 336 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 337 } 338 } 339 340 function void 341 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb) 342 { 343 b32 run_cuda_hilbert = 0; 344 b32 demodulate = 0; 345 346 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 347 switch (pb->pipeline.shaders[i]) { 348 case BeamformerShaderKind_CudaHilbert:{ run_cuda_hilbert = 1; }break; 349 case BeamformerShaderKind_Demodulate:{ demodulate = 1; }break; 350 default:{}break; 351 } 352 } 353 354 if (demodulate) run_cuda_hilbert = 0; 355 356 BeamformerDataKind data_kind = pb->pipeline.data_kind; 357 cp->iq_pipeline = beamformer_data_kind_complex[data_kind] || demodulate || run_cuda_hilbert; 358 359 f32 sampling_frequency = pb->parameters.sampling_frequency; 360 u32 decimation_rate = Max(pb->parameters.decimation_rate, 1); 361 u32 sample_count = pb->parameters.sample_count; 362 if (demodulate) { 363 sample_count /= (2 * decimation_rate); 364 sampling_frequency /= 2 * (f32)decimation_rate; 365 } 366 367 cp->rf_size = sample_count * pb->parameters.channel_count * pb->parameters.acquisition_count; 368 if (cp->iq_pipeline) cp->rf_size *= 8; 369 else cp->rf_size *= 4; 370 371 u32 das_sample_stride = 1; 372 u32 das_transmit_stride = sample_count; 373 u32 das_channel_stride = sample_count * pb->parameters.acquisition_count; 374 375 f32 time_offset = pb->parameters.time_offset; 376 377 // TODO(rnp): subgroup size 378 u32 subgroup_size = gl_parameters.vendor_id == GLVendor_NVIDIA ? 32 : 64; 379 380 cp->pipeline.shader_count = 0; 381 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 382 BeamformerShaderParameters *sp = pb->pipeline.parameters + i; 383 u32 slot = cp->pipeline.shader_count; 384 u32 shader = pb->pipeline.shaders[i]; 385 b32 commit = 0; 386 387 BeamformerShaderDescriptor *ld = cp->shader_descriptors + slot - 1; 388 BeamformerShaderDescriptor *sd = cp->shader_descriptors + slot; 389 zero_struct(sd); 390 391 switch (shader) { 392 case BeamformerShaderKind_CudaHilbert:{ commit = run_cuda_hilbert; }break; 393 case BeamformerShaderKind_Decode:{ 394 /* TODO(rnp): rework decode first and demodulate after */ 395 b32 first = slot == 0; 396 397 sd->bake.data_kind = data_kind; 398 if (!first) { 399 if (data_kind == BeamformerDataKind_Int16) { 400 sd->bake.data_kind = BeamformerDataKind_Int16Complex; 401 } else { 402 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 403 } 404 } 405 406 BeamformerShaderKind *last_shader = cp->pipeline.shaders + slot - 1; 407 assert(first || ((*last_shader == BeamformerShaderKind_Demodulate || 408 *last_shader == BeamformerShaderKind_Filter))); 409 410 BeamformerShaderDecodeBakeParameters *db = &sd->bake.Decode; 411 db->decode_mode = pb->parameters.decode_mode; 412 db->transmit_count = pb->parameters.acquisition_count; 413 414 u32 channel_stride = pb->parameters.acquisition_count * pb->parameters.sample_count; 415 db->input_sample_stride = first? 1 : ld->bake.Filter.output_sample_stride; 416 db->input_channel_stride = first? channel_stride : ld->bake.Filter.output_channel_stride; 417 db->input_transmit_stride = first? pb->parameters.sample_count : 1; 418 419 db->output_sample_stride = das_sample_stride; 420 db->output_channel_stride = das_channel_stride; 421 db->output_transmit_stride = das_transmit_stride; 422 if (first) { 423 db->output_channel_stride *= decimation_rate; 424 db->output_transmit_stride *= decimation_rate; 425 } 426 427 if (run_cuda_hilbert) sd->bake.flags |= BeamformerShaderDecodeFlags_DilateOutput; 428 429 if (db->decode_mode == BeamformerDecodeMode_None) { 430 sd->layout = (uv3){{subgroup_size, 1, 1}}; 431 432 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 433 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 434 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z); 435 } else if (db->transmit_count > 40) { 436 sd->bake.flags |= BeamformerShaderDecodeFlags_UseSharedMemory; 437 db->to_process = 2; 438 439 if (db->transmit_count == 48) 440 db->to_process = db->transmit_count / 16; 441 442 b32 use_16z = db->transmit_count == 48 || db->transmit_count == 80 || 443 db->transmit_count == 96 || db->transmit_count == 160; 444 sd->layout = (uv3){{4, 1, use_16z? 16 : 32}}; 445 446 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 447 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 448 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z / (f32)db->to_process); 449 } else { 450 db->to_process = 1; 451 452 /* NOTE(rnp): register caching. using more threads will cause the compiler to do 453 * contortions to avoid spilling registers. using less gives higher performance */ 454 sd->layout = (uv3){{subgroup_size / 2, 1, 1}}; 455 456 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 457 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 458 sd->dispatch.z = 1; 459 } 460 461 if (first) sd->dispatch.x *= decimation_rate; 462 463 /* NOTE(rnp): decode 2 samples per dispatch when data is i16 */ 464 if (first && data_kind == BeamformerDataKind_Int16) 465 sd->dispatch.x = (u32)ceil_f32((f32)sd->dispatch.x / 2); 466 467 commit = first || db->decode_mode != BeamformerDecodeMode_None; 468 }break; 469 case BeamformerShaderKind_Demodulate: 470 case BeamformerShaderKind_Filter: 471 { 472 b32 first = slot == 0; 473 b32 demod = shader == BeamformerShaderKind_Demodulate; 474 BeamformerFilter *f = cp->filters + sp->filter_slot; 475 476 time_offset += f->time_delay; 477 478 BeamformerShaderFilterBakeParameters *fb = &sd->bake.Filter; 479 fb->filter_length = (u32)f->length; 480 if (demod) sd->bake.flags |= BeamformerShaderFilterFlags_Demodulate; 481 if (f->parameters.complex) sd->bake.flags |= BeamformerShaderFilterFlags_ComplexFilter; 482 483 sd->bake.data_kind = data_kind; 484 if (!first) sd->bake.data_kind = BeamformerDataKind_Float32; 485 486 /* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating 487 * between sampling the I portion and the Q portion of an IQ signal. Therefore there 488 * is an implicit decimation factor of 2 which must always be included. All code here 489 * assumes that the signal was sampled in such a way that supports this operation. 490 * To recover IQ[n] from the sampled data (RF[n]) we do the following: 491 * I[n] = RF[n] 492 * Q[n] = RF[n + 1] 493 * IQ[n] = I[n] - j*Q[n] 494 */ 495 if (demod) { 496 fb->demodulation_frequency = pb->parameters.demodulation_frequency; 497 fb->sampling_frequency = pb->parameters.sampling_frequency / 2; 498 fb->decimation_rate = decimation_rate; 499 fb->sample_count = pb->parameters.sample_count; 500 501 fb->output_channel_stride = das_channel_stride; 502 fb->output_sample_stride = das_sample_stride; 503 fb->output_transmit_stride = das_transmit_stride; 504 505 if (first) { 506 fb->input_channel_stride = pb->parameters.sample_count * pb->parameters.acquisition_count / 2; 507 fb->input_sample_stride = 1; 508 fb->input_transmit_stride = pb->parameters.sample_count / 2; 509 510 if (pb->parameters.decode_mode == BeamformerDecodeMode_None) { 511 sd->bake.flags |= BeamformerShaderFilterFlags_OutputFloats; 512 } else { 513 /* NOTE(rnp): output optimized layout for decoding */ 514 fb->output_channel_stride = das_channel_stride; 515 fb->output_sample_stride = pb->parameters.acquisition_count; 516 fb->output_transmit_stride = 1; 517 } 518 } else { 519 assert(cp->pipeline.shaders[slot - 1] == BeamformerShaderKind_Decode); 520 fb->input_channel_stride = ld->bake.Decode.output_channel_stride; 521 fb->input_sample_stride = ld->bake.Decode.output_sample_stride; 522 fb->input_transmit_stride = ld->bake.Decode.output_transmit_stride; 523 } 524 } else { 525 fb->decimation_rate = 1; 526 fb->output_channel_stride = sample_count * pb->parameters.acquisition_count; 527 fb->output_sample_stride = 1; 528 fb->output_transmit_stride = sample_count; 529 fb->input_channel_stride = sample_count * pb->parameters.acquisition_count; 530 fb->input_sample_stride = 1; 531 fb->input_transmit_stride = sample_count; 532 fb->sample_count = sample_count; 533 } 534 535 /* TODO(rnp): filter may need a different dispatch layout */ 536 sd->layout = (uv3){{128, 1, 1}}; 537 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 538 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 539 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z); 540 541 commit = 1; 542 }break; 543 case BeamformerShaderKind_DAS:{ 544 sd->bake.data_kind = BeamformerDataKind_Float32; 545 if (cp->iq_pipeline) 546 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 547 548 BeamformerShaderDASBakeParameters *db = &sd->bake.DAS; 549 BeamformerDASUBO *du = &cp->das_ubo_data; 550 du->xdc_element_pitch = pb->parameters.xdc_element_pitch; 551 db->sampling_frequency = sampling_frequency; 552 db->demodulation_frequency = pb->parameters.demodulation_frequency; 553 db->speed_of_sound = pb->parameters.speed_of_sound; 554 db->time_offset = time_offset; 555 db->f_number = pb->parameters.f_number; 556 db->acquisition_kind = pb->parameters.acquisition_kind; 557 db->sample_count = sample_count; 558 db->channel_count = pb->parameters.channel_count; 559 db->acquisition_count = pb->parameters.acquisition_count; 560 db->interpolation_mode = pb->parameters.interpolation_mode; 561 db->transmit_angle = pb->parameters.focal_vector.E[0]; 562 db->focus_depth = pb->parameters.focal_vector.E[1]; 563 db->transmit_receive_orientation = pb->parameters.transmit_receive_orientation; 564 565 // NOTE(rnp): old gcc will miscompile an assignment 566 mem_copy(du->voxel_transform.E, pb->parameters.das_voxel_transform.E, sizeof(du->voxel_transform)); 567 mem_copy(du->xdc_transform.E, pb->parameters.xdc_transform.E, sizeof(du->xdc_transform)); 568 569 du->voxel_transform = m4_mul(cp->ui_voxel_transform, du->voxel_transform); 570 571 u32 id = pb->parameters.acquisition_kind; 572 573 if (id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_FORCES) 574 du->voxel_transform = m4_mul(du->xdc_transform, du->voxel_transform); 575 576 if (id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_UHERCULES) 577 sd->bake.flags |= BeamformerShaderDASFlags_Sparse; 578 579 if (pb->parameters.single_focus) sd->bake.flags |= BeamformerShaderDASFlags_SingleFocus; 580 if (pb->parameters.single_orientation) sd->bake.flags |= BeamformerShaderDASFlags_SingleOrientation; 581 if (pb->parameters.coherency_weighting) sd->bake.flags |= BeamformerShaderDASFlags_CoherencyWeighting; 582 else sd->bake.flags |= BeamformerShaderDASFlags_Fast; 583 584 sd->layout = (uv3){{1, 1, 1}}; 585 586 b32 has_x = cp->output_points.x > 1; 587 b32 has_y = cp->output_points.y > 1; 588 b32 has_z = cp->output_points.z > 1; 589 590 u32 grid_3d_z_size = Max(1, subgroup_size / (4 * 4)); 591 u32 grid_2d_y_size = Max(1, subgroup_size / 8); 592 593 switch (iv3_dimension(cp->output_points)) { 594 595 case 1:{ 596 if (has_x) sd->layout.x = subgroup_size; 597 if (has_y) sd->layout.y = subgroup_size; 598 if (has_z) sd->layout.z = subgroup_size; 599 }break; 600 601 case 2:{ 602 if (has_x && has_y) {sd->layout.x = 8; sd->layout.y = grid_2d_y_size;} 603 if (has_x && has_z) {sd->layout.x = 8; sd->layout.z = grid_2d_y_size;} 604 if (has_y && has_z) {sd->layout.y = 8; sd->layout.z = grid_2d_y_size;} 605 }break; 606 607 case 3:{sd->layout = (uv3){{4, 4, grid_3d_z_size}};}break; 608 609 InvalidDefaultCase; 610 } 611 612 sd->dispatch.x = (u32)ceil_f32((f32)cp->output_points.x / sd->layout.x); 613 sd->dispatch.y = (u32)ceil_f32((f32)cp->output_points.y / sd->layout.y); 614 sd->dispatch.z = (u32)ceil_f32((f32)cp->output_points.z / sd->layout.z); 615 616 commit = 1; 617 }break; 618 default:{ commit = 1; }break; 619 } 620 621 if (commit) { 622 u32 index = cp->pipeline.shader_count++; 623 cp->pipeline.shaders[index] = shader; 624 cp->pipeline.parameters[index] = *sp; 625 } 626 } 627 cp->pipeline.data_kind = data_kind; 628 } 629 630 function void 631 stream_push_shader_header(Stream *s, BeamformerShaderKind shader_kind, s8 header) 632 { 633 stream_append_s8s(s, s8("#version 460 core\n\n"), header); 634 635 switch (shader_kind) { 636 case BeamformerShaderKind_DAS:{ 637 stream_append_s8(s, s8("" 638 "layout(location = " str(DAS_CYCLE_T_UNIFORM_LOC) ") uniform uint u_cycle_t;\n" 639 "layout(location = " str(DAS_FAST_CHANNEL_UNIFORM_LOC) ") uniform int u_channel;\n\n" 640 )); 641 }break; 642 case BeamformerShaderKind_Decode:{ 643 stream_append_s8s(s, s8("" 644 "layout(location = " str(DECODE_FIRST_PASS_UNIFORM_LOC) ") uniform bool u_first_pass;\n\n" 645 )); 646 }break; 647 case BeamformerShaderKind_MinMax:{ 648 stream_append_s8(s, s8("layout(location = " str(MIN_MAX_MIPS_LEVEL_UNIFORM_LOC) 649 ") uniform int u_mip_map;\n\n")); 650 }break; 651 case BeamformerShaderKind_Sum:{ 652 stream_append_s8(s, s8("layout(location = " str(SUM_PRESCALE_UNIFORM_LOC) 653 ") uniform float u_sum_prescale = 1.0;\n\n")); 654 }break; 655 default:{}break; 656 } 657 } 658 659 function void 660 load_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 shader_slot, Arena arena) 661 { 662 read_only local_persist s8 compute_headers[BeamformerShaderKind_ComputeCount] = { 663 /* X(name, type, gltype) */ 664 #define X(name, t, gltype) "\t" #gltype " " #name ";\n" 665 [BeamformerShaderKind_DAS] = s8_comp("layout(std140, binding = 0) uniform parameters {\n" 666 BEAMFORMER_DAS_UBO_PARAM_LIST 667 "};\n\n" 668 ), 669 #undef X 670 }; 671 672 BeamformerShaderKind shader = cp->pipeline.shaders[shader_slot]; 673 674 u32 program = 0; 675 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[shader]; 676 if (reloadable_index != -1) { 677 BeamformerShaderKind base_shader = beamformer_reloadable_shader_kinds[reloadable_index]; 678 s8 path; 679 if (!BakeShaders) 680 path = push_s8_from_parts(&arena, os_path_separator(), s8("shaders"), 681 beamformer_reloadable_shader_files[reloadable_index]); 682 683 Stream shader_stream = arena_stream(arena); 684 stream_push_shader_header(&shader_stream, base_shader, compute_headers[base_shader]); 685 686 i32 header_vector_length = beamformer_shader_header_vector_lengths[reloadable_index]; 687 i32 *header_vector = beamformer_shader_header_vectors[reloadable_index]; 688 for (i32 index = 0; index < header_vector_length; index++) 689 stream_append_s8(&shader_stream, beamformer_shader_global_header_strings[header_vector[index]]); 690 691 BeamformerShaderDescriptor *sd = cp->shader_descriptors + shader_slot; 692 693 if (sd->layout.x != 0) { 694 stream_append_s8(&shader_stream, s8("layout(local_size_x = ")); 695 stream_append_u64(&shader_stream, sd->layout.x); 696 stream_append_s8(&shader_stream, s8(", local_size_y = ")); 697 stream_append_u64(&shader_stream, sd->layout.y); 698 stream_append_s8(&shader_stream, s8(", local_size_z = ")); 699 stream_append_u64(&shader_stream, sd->layout.z); 700 stream_append_s8(&shader_stream, s8(") in;\n\n")); 701 } 702 703 u32 *parameters = (u32 *)&sd->bake; 704 s8 *names = beamformer_shader_bake_parameter_names[reloadable_index]; 705 u32 float_bits = beamformer_shader_bake_parameter_float_bits[reloadable_index]; 706 i32 count = beamformer_shader_bake_parameter_counts[reloadable_index]; 707 708 for (i32 index = 0; index < count; index++) { 709 stream_append_s8s(&shader_stream, s8("#define "), names[index], 710 (float_bits & (1 << index))? s8(" uintBitsToFloat") : s8(" "), s8("(0x")); 711 stream_append_hex_u64(&shader_stream, parameters[index]); 712 stream_append_s8(&shader_stream, s8(")\n")); 713 } 714 715 stream_append_s8(&shader_stream, s8("#define DataKind (0x")); 716 stream_append_hex_u64(&shader_stream, sd->bake.data_kind); 717 stream_append_s8(&shader_stream, s8(")\n\n")); 718 719 s8 *flag_names = beamformer_shader_flag_strings[reloadable_index]; 720 u32 flag_count = beamformer_shader_flag_strings_count[reloadable_index]; 721 u32 flags = sd->bake.flags; 722 for (u32 bit = 0; bit < flag_count; bit++) { 723 stream_append_s8s(&shader_stream, s8("#define "), flag_names[bit], 724 (flags & (1 << bit))? s8(" 1") : s8(" 0"), s8("\n")); 725 } 726 727 if (!renderdoc_attached()) 728 stream_append_s8(&shader_stream, s8("\n#line 1\n")); 729 730 s8 shader_text; 731 if (BakeShaders) { 732 stream_append_s8(&shader_stream, beamformer_shader_data[reloadable_index]); 733 shader_text = arena_stream_commit(&arena, &shader_stream); 734 } else { 735 shader_text = arena_stream_commit(&arena, &shader_stream); 736 i64 length = os_read_entire_file((c8 *)path.data, arena.beg, arena_capacity(&arena, u8)); 737 shader_text.len += length; 738 arena_commit(&arena, length); 739 } 740 741 /* TODO(rnp): instance name */ 742 s8 shader_name = beamformer_shader_names[shader]; 743 program = load_shader(arena, &shader_text, (u32 []){GL_COMPUTE_SHADER}, 1, shader_name); 744 } 745 746 glDeleteProgram(cp->programs[shader_slot]); 747 cp->programs[shader_slot] = program; 748 } 749 750 function void 751 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena arena) 752 { 753 BeamformerParameterBlock *pb = beamformer_parameter_block_lock(ctx->shared_memory, block, -1); 754 for EachBit(pb->region_update_flags, region) { 755 switch (region) { 756 case BeamformerParameterRegionFlag_NotifyUI:{ 757 atomic_store_u32(&ctx->ui_dirty_parameter_blocks, 1u << block); 758 }break; 759 760 case BeamformerParameterRegionFlag_ComputePipeline: 761 case BeamformerParameterRegionFlag_Parameters: 762 { 763 cp->output_points = das_valid_points(pb->parameters.output_points.xyz); 764 cp->average_frames = pb->parameters.output_points.E[3]; 765 766 plan_compute_pipeline(cp, pb); 767 768 /* NOTE(rnp): these are both handled by plan_compute_pipeline() */ 769 u32 mask = 1 << BeamformerParameterBlockRegion_ComputePipeline | 770 1 << BeamformerParameterBlockRegion_Parameters; 771 pb->region_update_flags &= ~mask; 772 773 for (u32 shader_slot = 0; shader_slot < cp->pipeline.shader_count; shader_slot++) { 774 u128 hash = u128_hash_from_data(cp->shader_descriptors + shader_slot, sizeof(BeamformerShaderDescriptor)); 775 if (!u128_equal(hash, cp->shader_hashes[shader_slot])) 776 cp->dirty_programs |= 1 << shader_slot; 777 cp->shader_hashes[shader_slot] = hash; 778 } 779 780 #define X(k, t, v) glNamedBufferSubData(cp->ubos[BeamformerComputeUBOKind_##k], \ 781 0, sizeof(t), &cp->v ## _ubo_data); 782 BEAMFORMER_COMPUTE_UBO_LIST 783 #undef X 784 785 cp->acquisition_count = pb->parameters.acquisition_count; 786 cp->acquisition_kind = pb->parameters.acquisition_kind; 787 788 u32 decoded_data_size = cp->rf_size; 789 if (ctx->compute_context.ping_pong_ssbo_size < decoded_data_size) 790 alloc_shader_storage(ctx, decoded_data_size, arena); 791 792 if (cp->hadamard_order != (i32)cp->acquisition_count) 793 update_hadamard_texture(cp, (i32)cp->acquisition_count, 0, arena); 794 795 mem_copy(cp->voxel_transform.E, pb->parameters.das_voxel_transform.E, sizeof(cp->voxel_transform)); 796 797 GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F; 798 if (cp->average_frames > 1 && !beamformer_frame_compatible(ctx->averaged_frames + 0, cp->output_points, gl_kind)) { 799 alloc_beamform_frame(ctx->averaged_frames + 0, cp->output_points, gl_kind, s8("Averaged Frame"), arena); 800 alloc_beamform_frame(ctx->averaged_frames + 1, cp->output_points, gl_kind, s8("Averaged Frame"), arena); 801 } 802 }break; 803 case BeamformerParameterBlockRegion_ChannelMapping:{ 804 cuda_set_channel_mapping(pb->channel_mapping); 805 }break; 806 case BeamformerParameterRegionFlag_FocalVectors: 807 case BeamformerParameterRegionFlag_SparseElements: 808 case BeamformerParameterRegionFlag_TransmitReceiveOrientations: 809 { 810 BeamformerComputeTextureKind texture_kind = 0; 811 u32 pixel_type = 0, texture_format = 0; 812 switch (region) { 813 #define X(kind, _gl, tf, pt, ...) \ 814 case BeamformerParameterRegionFlag_##kind:{ \ 815 texture_kind = BeamformerComputeTextureKind_## kind; \ 816 texture_format = tf; \ 817 pixel_type = pt; \ 818 }break; 819 BEAMFORMER_COMPUTE_TEXTURE_LIST 820 #undef X 821 InvalidDefaultCase; 822 } 823 glTextureSubImage1D(cp->textures[texture_kind], 0, 0, BeamformerMaxChannelCount, 824 texture_format, pixel_type, 825 (u8 *)pb + BeamformerParameterBlockRegionOffsets[region]); 826 }break; 827 } 828 } 829 beamformer_parameter_block_unlock(ctx->shared_memory, block); 830 } 831 832 function void 833 do_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, BeamformerFrame *frame, 834 BeamformerShaderKind shader, u32 shader_slot, BeamformerShaderParameters *sp, Arena arena) 835 { 836 BeamformerComputeContext *cc = &ctx->compute_context; 837 838 u32 program = cp->programs[shader_slot]; 839 glUseProgram(program); 840 841 u32 output_ssbo_idx = !cc->last_output_ssbo_index; 842 u32 input_ssbo_idx = cc->last_output_ssbo_index; 843 844 uv3 dispatch = cp->shader_descriptors[shader_slot].dispatch; 845 switch (shader) { 846 case BeamformerShaderKind_Decode:{ 847 glBindImageTexture(0, cp->textures[BeamformerComputeTextureKind_Hadamard], 0, 0, 0, GL_READ_ONLY, GL_R16F); 848 849 BeamformerDecodeMode mode = cp->shader_descriptors[shader_slot].bake.Decode.decode_mode; 850 if (shader_slot == 0) { 851 if (mode != BeamformerDecodeMode_None) { 852 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[input_ssbo_idx]); 853 glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 1); 854 855 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 856 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 857 } 858 } 859 860 if (mode != BeamformerDecodeMode_None) 861 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]); 862 863 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cc->ping_pong_ssbos[output_ssbo_idx]); 864 865 glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 0); 866 867 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 868 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 869 870 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 871 }break; 872 case BeamformerShaderKind_CudaDecode:{ 873 cuda_decode(0, output_ssbo_idx, 0); 874 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 875 }break; 876 case BeamformerShaderKind_CudaHilbert:{ 877 cuda_hilbert(input_ssbo_idx, output_ssbo_idx); 878 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 879 }break; 880 case BeamformerShaderKind_Filter: 881 case BeamformerShaderKind_Demodulate: 882 { 883 if (shader_slot != 0) 884 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]); 885 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[output_ssbo_idx]); 886 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cp->filters[sp->filter_slot].ssbo); 887 888 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 889 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 890 891 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 892 }break; 893 case BeamformerShaderKind_MinMax:{ 894 for (i32 i = 1; i < frame->mips; i++) { 895 glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 896 glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F); 897 glProgramUniform1i(program, MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i); 898 899 u32 width = (u32)frame->dim.x >> i; 900 u32 height = (u32)frame->dim.y >> i; 901 u32 depth = (u32)frame->dim.z >> i; 902 glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32)); 903 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 904 } 905 }break; 906 case BeamformerShaderKind_DAS:{ 907 local_persist u32 das_cycle_t = 0; 908 909 BeamformerShaderBakeParameters *bp = &cp->shader_descriptors[shader_slot].bake; 910 b32 fast = (bp->flags & BeamformerShaderDASFlags_Fast) != 0; 911 b32 sparse = (bp->flags & BeamformerShaderDASFlags_Sparse) != 0; 912 913 if (fast) { 914 glClearTexImage(frame->texture, 0, GL_RED, GL_FLOAT, 0); 915 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 916 glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_READ_WRITE, cp->iq_pipeline ? GL_RG32F : GL_R32F); 917 } else { 918 glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_WRITE_ONLY, cp->iq_pipeline ? GL_RG32F : GL_R32F); 919 } 920 921 u32 sparse_texture = cp->textures[BeamformerComputeTextureKind_SparseElements]; 922 if (!sparse) sparse_texture = 0; 923 924 glBindBufferBase(GL_UNIFORM_BUFFER, 0, cp->ubos[BeamformerComputeUBOKind_DAS]); 925 glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx], 0, cp->rf_size); 926 glBindImageTexture(1, sparse_texture, 0, 0, 0, GL_READ_ONLY, GL_R16I); 927 glBindImageTexture(2, cp->textures[BeamformerComputeTextureKind_FocalVectors], 0, 0, 0, GL_READ_ONLY, GL_RG32F); 928 glBindImageTexture(3, cp->textures[BeamformerComputeTextureKind_TransmitReceiveOrientations], 0, 0, 0, GL_READ_ONLY, GL_R8I); 929 930 glProgramUniform1ui(program, DAS_CYCLE_T_UNIFORM_LOC, das_cycle_t++); 931 932 if (fast) { 933 i32 loop_end; 934 if (bp->DAS.acquisition_kind == BeamformerAcquisitionKind_RCA_VLS || 935 bp->DAS.acquisition_kind == BeamformerAcquisitionKind_RCA_TPW) 936 { 937 /* NOTE(rnp): to avoid repeatedly sampling the whole focal vectors 938 * texture we loop over transmits for VLS/TPW */ 939 loop_end = (i32)bp->DAS.acquisition_count; 940 } else { 941 loop_end = (i32)bp->DAS.channel_count; 942 } 943 f32 percent_per_step = 1.0f / (f32)loop_end; 944 cc->processing_progress = -percent_per_step; 945 for (i32 index = 0; index < loop_end; index++) { 946 cc->processing_progress += percent_per_step; 947 /* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */ 948 glFinish(); 949 glProgramUniform1i(program, DAS_FAST_CHANNEL_UNIFORM_LOC, index); 950 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 951 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 952 } 953 } else { 954 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 955 } 956 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT|GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 957 }break; 958 case BeamformerShaderKind_Sum:{ 959 u32 aframe_index = ctx->averaged_frame_index % ARRAY_COUNT(ctx->averaged_frames); 960 BeamformerFrame *aframe = ctx->averaged_frames + aframe_index; 961 aframe->id = ctx->averaged_frame_index; 962 atomic_store_u32(&aframe->ready_to_present, 0); 963 /* TODO(rnp): hack we need a better way of specifying which frames to sum; 964 * this is fine for rolling averaging but what if we want to do something else */ 965 assert(frame >= ctx->beamform_frames); 966 assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames)); 967 u32 base_index = (u32)(frame - ctx->beamform_frames); 968 u32 to_average = (u32)cp->average_frames; 969 u32 frame_count = 0; 970 u32 *in_textures = push_array(&arena, u32, BeamformerMaxSavedFrames); 971 ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average); 972 for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi)) 973 in_textures[frame_count++] = it->texture; 974 975 assert(to_average == frame_count); 976 977 glProgramUniform1f(program, SUM_PRESCALE_UNIFORM_LOC, 1 / (f32)frame_count); 978 do_sum_shader(cc, in_textures, frame_count, aframe->texture, aframe->dim); 979 mem_copy(aframe->voxel_transform.E, frame->voxel_transform.E, sizeof(frame->voxel_transform)); 980 aframe->compound_count = frame->compound_count; 981 aframe->acquisition_kind = frame->acquisition_kind; 982 }break; 983 InvalidDefaultCase; 984 } 985 } 986 987 function s8 988 shader_text_with_header(s8 header, s8 filepath, b32 has_file, BeamformerShaderKind shader_kind, Arena *arena) 989 { 990 Stream sb = arena_stream(*arena); 991 stream_push_shader_header(&sb, shader_kind, header); 992 stream_append_s8(&sb, s8("\n#line 1\n")); 993 994 s8 result; 995 if (BakeShaders) { 996 /* TODO(rnp): better handling of shaders with no backing file */ 997 if (has_file) { 998 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[shader_kind]; 999 stream_append_s8(&sb, beamformer_shader_data[reloadable_index]); 1000 } 1001 result = arena_stream_commit(arena, &sb); 1002 } else { 1003 result = arena_stream_commit(arena, &sb); 1004 if (has_file) { 1005 i64 length = os_read_entire_file((c8 *)filepath.data, arena->beg, arena_capacity(arena, u8)); 1006 result.len += length; 1007 arena_commit(arena, length); 1008 } 1009 } 1010 1011 return result; 1012 } 1013 1014 /* NOTE(rnp): currently this function is only handling rendering shaders. 1015 * look at load_compute_shader for compute shaders */ 1016 function void 1017 beamformer_reload_shader(BeamformerCtx *ctx, BeamformerShaderReloadContext *src, Arena arena, s8 shader_name) 1018 { 1019 BeamformerShaderKind kind = beamformer_reloadable_shader_kinds[src->reloadable_info_index]; 1020 assert(kind == BeamformerShaderKind_Render3D); 1021 1022 s8 path = push_s8_from_parts(&arena, os_path_separator(), s8("shaders"), 1023 beamformer_reloadable_shader_files[src->reloadable_info_index]); 1024 1025 i32 shader_count = 1; 1026 BeamformerShaderReloadContext *link = src->link; 1027 while (link != src) { shader_count++; link = link->link; } 1028 1029 s8 *shader_texts = push_array(&arena, s8, shader_count); 1030 u32 *shader_types = push_array(&arena, u32, shader_count); 1031 1032 i32 index = 0; 1033 do { 1034 b32 has_file = link->reloadable_info_index >= 0; 1035 shader_texts[index] = shader_text_with_header(link->header, path, has_file, kind, &arena); 1036 shader_types[index] = link->gl_type; 1037 index++; 1038 link = link->link; 1039 } while (link != src); 1040 1041 u32 *shader = &ctx->frame_view_render_context.shader; 1042 glDeleteProgram(*shader); 1043 *shader = load_shader(arena, shader_texts, shader_types, shader_count, shader_name); 1044 ctx->frame_view_render_context.updated = 1; 1045 } 1046 1047 function void 1048 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena, iptr gl_context) 1049 { 1050 BeamformerComputeContext * cs = &ctx->compute_context; 1051 BeamformerSharedMemory * sm = ctx->shared_memory; 1052 1053 BeamformWork *work = beamform_work_queue_pop(q); 1054 while (work) { 1055 b32 can_commit = 1; 1056 switch (work->kind) { 1057 case BeamformerWorkKind_ExportBuffer:{ 1058 /* TODO(rnp): better way of handling DispatchCompute barrier */ 1059 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute); 1060 beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)work->lock, (u32)-1); 1061 BeamformerExportContext *ec = &work->export_context; 1062 switch (ec->kind) { 1063 case BeamformerExportKind_BeamformedData:{ 1064 BeamformerFrame *frame = ctx->latest_frame; 1065 if (frame) { 1066 assert(frame->ready_to_present); 1067 u32 texture = frame->texture; 1068 iv3 dim = frame->dim; 1069 u32 out_size = (u32)dim.x * (u32)dim.y * (u32)dim.z * 2 * sizeof(f32); 1070 if (out_size <= ec->size) { 1071 glGetTextureImage(texture, 0, GL_RG, GL_FLOAT, (i32)out_size, 1072 beamformer_shared_memory_scratch_arena(sm, ctx->shared_memory_size).beg); 1073 } 1074 } 1075 }break; 1076 case BeamformerExportKind_Stats:{ 1077 ComputeTimingTable *table = ctx->compute_timing_table; 1078 /* NOTE(rnp): do a little spin to let this finish updating */ 1079 spin_wait(table->write_index != atomic_load_u32(&table->read_index)); 1080 ComputeShaderStats *stats = ctx->compute_shader_stats; 1081 if (sizeof(stats->table) <= ec->size) 1082 mem_copy(beamformer_shared_memory_scratch_arena(sm, ctx->shared_memory_size).beg, 1083 &stats->table, sizeof(stats->table)); 1084 }break; 1085 InvalidDefaultCase; 1086 } 1087 beamformer_shared_memory_release_lock(ctx->shared_memory, work->lock); 1088 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync); 1089 }break; 1090 case BeamformerWorkKind_CreateFilter:{ 1091 /* TODO(rnp): this should probably get deleted and moved to lazy loading */ 1092 BeamformerCreateFilterContext *fctx = &work->create_filter_context; 1093 u32 block = fctx->parameter_block; 1094 u32 slot = fctx->filter_slot; 1095 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena); 1096 beamformer_filter_update(cp->filters + slot, fctx->parameters, block, slot, *arena); 1097 }break; 1098 case BeamformerWorkKind_ComputeIndirect:{ 1099 fill_frame_compute_work(ctx, work, work->compute_indirect_context.view_plane, 1100 work->compute_indirect_context.parameter_block, 1); 1101 } /* FALLTHROUGH */ 1102 case BeamformerWorkKind_Compute:{ 1103 DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[0], GL_RG32F, GL_RG, GL_FLOAT, 0);) 1104 DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[1], GL_RG32F, GL_RG, GL_FLOAT, 0);) 1105 DEBUG_DECL(glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);) 1106 1107 push_compute_timing_info(ctx->compute_timing_table, 1108 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin}); 1109 1110 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_block, arena); 1111 if (beamformer_parameter_block_dirty(sm, work->compute_context.parameter_block)) { 1112 u32 block = work->compute_context.parameter_block; 1113 beamformer_commit_parameter_block(ctx, cp, block, *arena); 1114 } 1115 1116 post_sync_barrier(ctx->shared_memory, work->lock); 1117 1118 u32 dirty_programs = atomic_swap_u32(&cp->dirty_programs, 0); 1119 static_assert(ISPOWEROF2(BeamformerMaxComputeShaderStages), 1120 "max compute shader stages must be power of 2"); 1121 assert((dirty_programs & ~((u32)BeamformerMaxComputeShaderStages - 1)) == 0); 1122 for EachBit(dirty_programs, slot) 1123 load_compute_shader(ctx, cp, (u32)slot, *arena); 1124 1125 atomic_store_u32(&cs->processing_compute, 1); 1126 start_renderdoc_capture(gl_context); 1127 1128 BeamformerFrame *frame = work->compute_context.frame; 1129 1130 GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F; 1131 if (!beamformer_frame_compatible(frame, cp->output_points, gl_kind)) 1132 alloc_beamform_frame(frame, cp->output_points, gl_kind, s8("Beamformed_Data"), *arena); 1133 1134 m4 voxel_transform = m4_mul(cp->ui_voxel_transform, cp->voxel_transform); 1135 mem_copy(frame->voxel_transform.E, voxel_transform.E, sizeof(voxel_transform)); 1136 frame->acquisition_kind = cp->acquisition_kind; 1137 frame->compound_count = cp->acquisition_count; 1138 1139 BeamformerComputeContext *cc = &ctx->compute_context; 1140 BeamformerComputePipeline *pipeline = &cp->pipeline; 1141 /* NOTE(rnp): first stage requires access to raw data buffer directly so we break 1142 * it out into a separate step. This way data can get released as soon as possible */ 1143 if (pipeline->shader_count > 0) { 1144 BeamformerRFBuffer *rf = &cs->rf_buffer; 1145 u32 slot = rf->compute_index % countof(rf->compute_syncs); 1146 1147 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1148 /* NOTE(rnp): compute indirect is used when uploading data. if compute thread 1149 * preempts upload it must wait for the fence to exist. then it must tell the 1150 * GPU to wait for upload to complete before it can start compute */ 1151 spin_wait(!atomic_load_u64(rf->upload_syncs + slot)); 1152 1153 glWaitSync(rf->upload_syncs[slot], 0, GL_TIMEOUT_IGNORED); 1154 glDeleteSync(rf->upload_syncs[slot]); 1155 rf->compute_index++; 1156 } else { 1157 slot = (rf->compute_index - 1) % countof(rf->compute_syncs); 1158 } 1159 1160 glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, rf->ssbo, slot * rf->active_rf_size, rf->active_rf_size); 1161 1162 glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[0]); 1163 do_compute_shader(ctx, cp, frame, pipeline->shaders[0], 0, pipeline->parameters + 0, *arena); 1164 glEndQuery(GL_TIME_ELAPSED); 1165 1166 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1167 atomic_store_u64(rf->compute_syncs + slot, glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)); 1168 atomic_store_u64(rf->upload_syncs + slot, 0); 1169 } 1170 } 1171 1172 b32 did_sum_shader = 0; 1173 for (u32 i = 1; i < pipeline->shader_count; i++) { 1174 did_sum_shader |= pipeline->shaders[i] == BeamformerShaderKind_Sum; 1175 glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[i]); 1176 do_compute_shader(ctx, cp, frame, pipeline->shaders[i], i, pipeline->parameters + i, *arena); 1177 glEndQuery(GL_TIME_ELAPSED); 1178 } 1179 1180 /* NOTE(rnp): the first of these blocks until work completes */ 1181 for (u32 i = 0; i < pipeline->shader_count; i++) { 1182 ComputeTimingInfo info = {0}; 1183 info.kind = ComputeTimingInfoKind_Shader; 1184 info.shader = pipeline->shaders[i]; 1185 glGetQueryObjectui64v(cc->shader_timer_ids[i], GL_QUERY_RESULT, &info.timer_count); 1186 push_compute_timing_info(ctx->compute_timing_table, info); 1187 } 1188 cs->processing_progress = 1; 1189 1190 frame->ready_to_present = 1; 1191 if (did_sum_shader) { 1192 u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames)); 1193 ctx->averaged_frames[aframe_index].view_plane_tag = frame->view_plane_tag; 1194 ctx->averaged_frames[aframe_index].ready_to_present = 1; 1195 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index)); 1196 } else { 1197 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame); 1198 } 1199 cs->processing_compute = 0; 1200 1201 push_compute_timing_info(ctx->compute_timing_table, 1202 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd}); 1203 1204 end_renderdoc_capture(gl_context); 1205 }break; 1206 InvalidDefaultCase; 1207 } 1208 1209 if (can_commit) { 1210 beamform_work_queue_pop_commit(q); 1211 work = beamform_work_queue_pop(q); 1212 } 1213 } 1214 } 1215 1216 function void 1217 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats) 1218 { 1219 /* TODO(rnp): we do not currently do anything to handle the potential for a half written 1220 * info item. this could result in garbage entries but they shouldn't really matter */ 1221 1222 u32 target = atomic_load_u32(&t->write_index); 1223 u32 stats_index = (stats->latest_frame_index + 1) % countof(stats->table.times); 1224 1225 static_assert(BeamformerShaderKind_Count + 1 <= 32, "timing coalescence bitfield test"); 1226 u32 seen_info_test = 0; 1227 1228 while (t->read_index != target) { 1229 ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)]; 1230 switch (info.kind) { 1231 case ComputeTimingInfoKind_ComputeFrameBegin:{ 1232 assert(t->compute_frame_active == 0); 1233 t->compute_frame_active = 1; 1234 /* NOTE(rnp): allow multiple instances of same shader to accumulate */ 1235 mem_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index])); 1236 }break; 1237 case ComputeTimingInfoKind_ComputeFrameEnd:{ 1238 assert(t->compute_frame_active == 1); 1239 t->compute_frame_active = 0; 1240 stats->latest_frame_index = stats_index; 1241 stats_index = (stats_index + 1) % countof(stats->table.times); 1242 }break; 1243 case ComputeTimingInfoKind_Shader:{ 1244 stats->table.times[stats_index][info.shader] += (f32)info.timer_count / 1.0e9f; 1245 seen_info_test |= (1u << info.shader); 1246 }break; 1247 case ComputeTimingInfoKind_RF_Data:{ 1248 stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas); 1249 f32 delta = (f32)(info.timer_count - stats->last_rf_timer_count) / 1.0e9f; 1250 stats->table.rf_time_deltas[stats->latest_rf_index] = delta; 1251 stats->last_rf_timer_count = info.timer_count; 1252 seen_info_test |= (1 << BeamformerShaderKind_Count); 1253 }break; 1254 } 1255 /* NOTE(rnp): do this at the end so that stats table is always in a consistent state */ 1256 atomic_add_u32(&t->read_index, 1); 1257 } 1258 1259 if (seen_info_test) { 1260 for EachEnumValue(BeamformerShaderKind, shader) { 1261 if (seen_info_test & (1 << shader)) { 1262 f32 sum = 0; 1263 for EachElement(stats->table.times, i) 1264 sum += stats->table.times[i][shader]; 1265 stats->average_times[shader] = sum / countof(stats->table.times); 1266 } 1267 } 1268 1269 if (seen_info_test & (1 << BeamformerShaderKind_Count)) { 1270 f32 sum = 0; 1271 for EachElement(stats->table.rf_time_deltas, i) 1272 sum += stats->table.rf_time_deltas[i]; 1273 stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas); 1274 } 1275 } 1276 } 1277 1278 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute) 1279 { 1280 BeamformerCtx *ctx = (BeamformerCtx *)user_context; 1281 BeamformerSharedMemory *sm = ctx->shared_memory; 1282 complete_queue(ctx, &sm->external_work_queue, arena, gl_context); 1283 complete_queue(ctx, ctx->beamform_work_queue, arena, gl_context); 1284 } 1285 1286 function void 1287 beamformer_rf_buffer_allocate(BeamformerRFBuffer *rf, u32 rf_size, b32 nvidia) 1288 { 1289 assert((rf_size % 64) == 0); 1290 if (!nvidia) glUnmapNamedBuffer(rf->ssbo); 1291 glDeleteBuffers(1, &rf->ssbo); 1292 glCreateBuffers(1, &rf->ssbo); 1293 1294 u32 buffer_flags = GL_DYNAMIC_STORAGE_BIT; 1295 if (!nvidia) buffer_flags |= GL_MAP_PERSISTENT_BIT|GL_MAP_WRITE_BIT; 1296 1297 glNamedBufferStorage(rf->ssbo, countof(rf->compute_syncs) * rf_size, 0, buffer_flags); 1298 1299 if (!nvidia) { 1300 u32 access = GL_MAP_PERSISTENT_BIT|GL_MAP_WRITE_BIT|GL_MAP_FLUSH_EXPLICIT_BIT|GL_MAP_UNSYNCHRONIZED_BIT; 1301 rf->buffer = glMapNamedBufferRange(rf->ssbo, 0, (GLsizei)(countof(rf->compute_syncs) * rf_size), access); 1302 } 1303 1304 LABEL_GL_OBJECT(GL_BUFFER, rf->ssbo, s8("Raw_RF_SSBO")); 1305 rf->size = rf_size; 1306 } 1307 1308 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload) 1309 { 1310 BeamformerSharedMemory *sm = ctx->shared_memory; 1311 BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace; 1312 BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF; 1313 1314 u64 rf_block_rf_size; 1315 if (atomic_load_u32(sm->locks + upload_lock) && 1316 (rf_block_rf_size = atomic_swap_u64(&sm->rf_block_rf_size, 0))) 1317 { 1318 beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)scratch_lock, (u32)-1); 1319 1320 BeamformerRFBuffer *rf = ctx->rf_buffer; 1321 BeamformerParameterBlock *b = beamformer_parameter_block(sm, (u32)(rf_block_rf_size >> 32ULL)); 1322 BeamformerParameters *bp = &b->parameters; 1323 BeamformerDataKind data_kind = b->pipeline.data_kind; 1324 1325 b32 nvidia = gl_parameters.vendor_id == GLVendor_NVIDIA; 1326 1327 rf->active_rf_size = (u32)round_up_to(rf_block_rf_size & 0xFFFFFFFFULL, 64); 1328 if unlikely(rf->size < rf->active_rf_size) 1329 beamformer_rf_buffer_allocate(rf, rf->active_rf_size, nvidia); 1330 1331 u32 slot = rf->insertion_index++ % countof(rf->compute_syncs); 1332 1333 /* NOTE(rnp): if the rest of the code is functioning then the first 1334 * time the compute thread processes an upload it must have gone 1335 * through this path. therefore it is safe to spin until it gets processed */ 1336 spin_wait(atomic_load_u64(rf->upload_syncs + slot)); 1337 1338 if (atomic_load_u64(rf->compute_syncs + slot)) { 1339 GLenum sync_result = glClientWaitSync(rf->compute_syncs[slot], 0, 1000000000); 1340 if (sync_result == GL_TIMEOUT_EXPIRED || sync_result == GL_WAIT_FAILED) { 1341 // TODO(rnp): what do? 1342 } 1343 glDeleteSync(rf->compute_syncs[slot]); 1344 } 1345 1346 u32 size = bp->channel_count * bp->acquisition_count * bp->sample_count * beamformer_data_kind_byte_size[data_kind]; 1347 u8 *data = beamformer_shared_memory_scratch_arena(sm, ctx->shared_memory_size).beg; 1348 1349 if (nvidia) glNamedBufferSubData(rf->ssbo, slot * rf->active_rf_size, (i32)size, data); 1350 else memory_copy_non_temporal(rf->buffer + slot * rf->active_rf_size, data, size); 1351 store_fence(); 1352 1353 beamformer_shared_memory_release_lock(ctx->shared_memory, (i32)scratch_lock); 1354 post_sync_barrier(ctx->shared_memory, upload_lock); 1355 1356 if (!nvidia) 1357 glFlushMappedNamedBufferRange(rf->ssbo, slot * rf->active_rf_size, (i32)rf->active_rf_size); 1358 1359 atomic_store_u64(rf->upload_syncs + slot, glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)); 1360 atomic_store_u64(rf->compute_syncs + slot, 0); 1361 1362 os_wake_all_waiters(ctx->compute_worker_sync); 1363 1364 ComputeTimingInfo info = {.kind = ComputeTimingInfoKind_RF_Data}; 1365 glGetQueryObjectui64v(rf->data_timestamp_query, GL_QUERY_RESULT, &info.timer_count); 1366 glQueryCounter(rf->data_timestamp_query, GL_TIMESTAMP); 1367 push_compute_timing_info(ctx->compute_timing_table, info); 1368 } 1369 } 1370 1371 function void 1372 beamformer_queue_compute(BeamformerCtx *ctx, BeamformerFrame *frame, u32 parameter_block) 1373 { 1374 BeamformerSharedMemory *sm = ctx->shared_memory; 1375 BeamformerSharedMemoryLockKind dispatch_lock = BeamformerSharedMemoryLockKind_DispatchCompute; 1376 if (!sm->live_imaging_parameters.active && beamformer_shared_memory_take_lock(sm, (i32)dispatch_lock, 0)) 1377 { 1378 BeamformWork *work = beamform_work_queue_push(ctx->beamform_work_queue); 1379 BeamformerViewPlaneTag tag = frame ? frame->view_plane_tag : 0; 1380 if (fill_frame_compute_work(ctx, work, tag, parameter_block, 0)) 1381 beamform_work_queue_push_commit(ctx->beamform_work_queue); 1382 } 1383 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1384 } 1385 1386 #include "ui.c" 1387 1388 function void 1389 beamformer_process_input_events(BeamformerCtx *ctx, BeamformerInput *input, 1390 BeamformerInputEvent *events, u32 event_count) 1391 { 1392 for (u32 index = 0; index < event_count; index++) { 1393 BeamformerInputEvent *event = events + index; 1394 switch (event->kind) { 1395 1396 case BeamformerInputEventKind_ExecutableReload:{ 1397 ui_init(ctx, ctx->ui_backing_store); 1398 1399 #if BEAMFORMER_RENDERDOC_HOOKS 1400 start_frame_capture = input->renderdoc_start_frame_capture; 1401 end_frame_capture = input->renderdoc_end_frame_capture; 1402 #endif 1403 }break; 1404 1405 case BeamformerInputEventKind_FileEvent:{ 1406 BeamformerFileReloadContext *frc = event->file_watch_user_context; 1407 switch (frc->kind) { 1408 case BeamformerFileReloadKind_Shader:{ 1409 BeamformerShaderReloadContext *src = frc->shader_reload_context; 1410 BeamformerShaderKind kind = beamformer_reloadable_shader_kinds[src->reloadable_info_index]; 1411 beamformer_reload_shader(ctx, src, ctx->arena, beamformer_shader_names[kind]); 1412 }break; 1413 case BeamformerFileReloadKind_ComputeShader:{ 1414 for EachElement(ctx->compute_context.compute_plans, block) { 1415 BeamformerComputePlan *cp = ctx->compute_context.compute_plans[block]; 1416 for (u32 slot = 0; cp && slot < cp->pipeline.shader_count; slot++) { 1417 i32 shader_index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1418 if (beamformer_reloadable_shader_kinds[shader_index] == frc->compute_shader_kind) 1419 atomic_or_u32(&cp->dirty_programs, 1 << slot); 1420 } 1421 } 1422 1423 if (ctx->latest_frame) 1424 beamformer_queue_compute(ctx, ctx->latest_frame, ctx->latest_frame->parameter_block); 1425 }break; 1426 InvalidDefaultCase; 1427 } 1428 }break; 1429 1430 InvalidDefaultCase; 1431 } 1432 } 1433 } 1434 1435 BEAMFORMER_EXPORT void 1436 beamformer_frame_step(BeamformerInput *input) 1437 { 1438 BeamformerCtx *ctx = BeamformerContextMemory(input->memory); 1439 1440 u64 current_time = os_timer_count(); 1441 dt_for_frame = (f64)(current_time - ctx->frame_timestamp) / os_system_info()->timer_frequency; 1442 ctx->frame_timestamp = current_time; 1443 1444 if (IsWindowResized()) { 1445 ctx->window_size.h = GetScreenHeight(); 1446 ctx->window_size.w = GetScreenWidth(); 1447 } 1448 1449 coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats); 1450 1451 beamformer_process_input_events(ctx, input, input->event_queue, input->event_count); 1452 1453 BeamformerSharedMemory *sm = ctx->shared_memory; 1454 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_UploadRF)) 1455 os_wake_all_waiters(&ctx->upload_worker.sync_variable); 1456 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_DispatchCompute)) 1457 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1458 1459 BeamformerFrame *frame = ctx->latest_frame; 1460 BeamformerViewPlaneTag tag = frame? frame->view_plane_tag : 0; 1461 draw_ui(ctx, input, frame, tag); 1462 1463 ctx->frame_view_render_context.updated = 0; 1464 }