beamformer.c (59678B)
1 /* See LICENSE for license details. */ 2 /* TODO(rnp): 3 * [ ]: refactor: replace UploadRF with just the scratch_rf_size variable, 4 * use below to spin wait in library 5 * [ ]: utilize umonitor/umwait (intel), monitorx/mwaitx (amd), and wfe/sev (aarch64) 6 * for power efficient low latency waiting 7 * [ ]: refactor: split decode into reshape and decode 8 * - the check for first pass reshaping is the last non constant check 9 * in the shader 10 * - this will also remove the need for the channel mapping in the decode shader 11 * [X]: refactor: ui: reload only shader which is affected by the interaction 12 * [ ]: BeamformWorkQueue -> BeamformerWorkQueue 13 * [ ]: need to keep track of gpu memory in some way 14 * - want to be able to store more than 16 2D frames but limit 3D frames 15 * - maybe keep track of how much gpu memory is committed for beamformed images 16 * and use that to determine when to loop back over existing textures 17 * - to do this maybe use a circular linked list instead of a flat array 18 * - then have a way of querying how many frames are available for a specific point count 19 * [ ]: bug: reinit cuda on hot-reload 20 */ 21 22 #include "beamformer.h" 23 24 global f32 dt_for_frame; 25 26 #define DECODE_FIRST_PASS_UNIFORM_LOC 1 27 28 #define DAS_LOCAL_SIZE_X 16 29 #define DAS_LOCAL_SIZE_Y 1 30 #define DAS_LOCAL_SIZE_Z 16 31 32 #define DAS_VOXEL_OFFSET_UNIFORM_LOC 2 33 #define DAS_CYCLE_T_UNIFORM_LOC 3 34 #define DAS_FAST_CHANNEL_UNIFORM_LOC 4 35 36 #define MIN_MAX_MIPS_LEVEL_UNIFORM_LOC 1 37 #define SUM_PRESCALE_UNIFORM_LOC 1 38 39 #ifndef _DEBUG 40 #define start_renderdoc_capture(...) 41 #define end_renderdoc_capture(...) 42 #else 43 global renderdoc_start_frame_capture_fn *start_frame_capture; 44 global renderdoc_end_frame_capture_fn *end_frame_capture; 45 #define start_renderdoc_capture(gl) if (start_frame_capture) start_frame_capture(gl, 0) 46 #define end_renderdoc_capture(gl) if (end_frame_capture) end_frame_capture(gl, 0) 47 #endif 48 49 typedef struct { 50 BeamformerFrame *frames; 51 u32 capacity; 52 u32 offset; 53 u32 cursor; 54 u32 needed_frames; 55 } ComputeFrameIterator; 56 57 function void 58 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block) 59 { 60 assert(block < countof(cc->compute_plans)); 61 BeamformerComputePlan *cp = cc->compute_plans[block]; 62 if (cp) { 63 glDeleteBuffers(countof(cp->ubos), cp->ubos); 64 glDeleteTextures(countof(cp->textures), cp->textures); 65 for (u32 i = 0; i < countof(cp->filters); i++) 66 glDeleteBuffers(1, &cp->filters[i].ssbo); 67 cc->compute_plans[block] = 0; 68 SLLPushFreelist(cp, cc->compute_plan_freelist); 69 } 70 } 71 72 function BeamformerComputePlan * 73 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena) 74 { 75 assert(block < countof(cc->compute_plans)); 76 BeamformerComputePlan *result = cc->compute_plans[block]; 77 if (!result) { 78 result = SLLPopFreelist(cc->compute_plan_freelist); 79 if (result) zero_struct(result); 80 else result = push_struct(arena, BeamformerComputePlan); 81 cc->compute_plans[block] = result; 82 83 glCreateBuffers(countof(result->ubos), result->ubos); 84 85 Stream label = arena_stream(*arena); 86 #define X(k, t, ...) \ 87 glNamedBufferStorage(result->ubos[BeamformerComputeUBOKind_##k], sizeof(t), \ 88 0, GL_DYNAMIC_STORAGE_BIT); \ 89 stream_append_s8(&label, s8(#t "[")); \ 90 stream_append_u64(&label, block); \ 91 stream_append_byte(&label, ']'); \ 92 glObjectLabel(GL_BUFFER, result->ubos[BeamformerComputeUBOKind_##k], \ 93 label.widx, (c8 *)label.data); \ 94 label.widx = 0; 95 BEAMFORMER_COMPUTE_UBO_LIST 96 #undef X 97 98 #define X(_k, t, ...) t, 99 GLenum gl_kind[] = {BEAMFORMER_COMPUTE_TEXTURE_LIST_FULL}; 100 #undef X 101 read_only local_persist s8 tex_prefix[] = { 102 #define X(k, ...) s8_comp(#k "["), 103 BEAMFORMER_COMPUTE_TEXTURE_LIST_FULL 104 #undef X 105 }; 106 glCreateTextures(GL_TEXTURE_1D, BeamformerComputeTextureKind_Count - 1, result->textures); 107 for (u32 i = 0; i < BeamformerComputeTextureKind_Count - 1; i++) { 108 /* TODO(rnp): this could be predicated on channel count for this compute plan */ 109 glTextureStorage1D(result->textures[i], 1, gl_kind[i], BeamformerMaxChannelCount); 110 stream_append_s8(&label, tex_prefix[i]); 111 stream_append_u64(&label, block); 112 stream_append_byte(&label, ']'); 113 glObjectLabel(GL_TEXTURE, result->textures[i], label.widx, (c8 *)label.data); 114 label.widx = 0; 115 } 116 } 117 return result; 118 } 119 120 function void 121 beamformer_filter_update(BeamformerFilter *f, BeamformerFilterKind kind, 122 BeamformerFilterParameters fp, u32 block, u32 slot, Arena arena) 123 { 124 #define X(k, ...) s8_comp(#k "Filter"), 125 read_only local_persist s8 filter_kinds[] = {BEAMFORMER_FILTER_KIND_LIST(,)}; 126 #undef X 127 128 Stream sb = arena_stream(arena); 129 stream_append_s8s(&sb, filter_kinds[kind % countof(filter_kinds)], s8("[")); 130 stream_append_u64(&sb, block); 131 stream_append_s8(&sb, s8("][")); 132 stream_append_u64(&sb, slot); 133 stream_append_byte(&sb, ']'); 134 s8 label = arena_stream_commit(&arena, &sb); 135 136 void *filter = 0; 137 switch (kind) { 138 case BeamformerFilterKind_Kaiser:{ 139 /* TODO(rnp): this should also support complex */ 140 /* TODO(rnp): implement this as an IFIR filter instead to reduce computation */ 141 filter = kaiser_low_pass_filter(&arena, fp.Kaiser.cutoff_frequency, fp.sampling_frequency, 142 fp.Kaiser.beta, (i32)fp.Kaiser.length); 143 f->length = (i32)fp.Kaiser.length; 144 f->time_delay = (f32)f->length / 2.0f / fp.sampling_frequency; 145 }break; 146 case BeamformerFilterKind_MatchedChirp:{ 147 typeof(fp.MatchedChirp) *mc = &fp.MatchedChirp; 148 f32 fs = fp.sampling_frequency; 149 f->length = (i32)(mc->duration * fs); 150 if (fp.complex) { 151 filter = baseband_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1, 0.5f); 152 f->time_delay = complex_filter_first_moment(filter, f->length, fs); 153 } else { 154 filter = rf_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1); 155 f->time_delay = real_filter_first_moment(filter, f->length, fs); 156 } 157 }break; 158 InvalidDefaultCase; 159 } 160 161 f->kind = kind; 162 f->parameters = fp; 163 164 glDeleteBuffers(1, &f->ssbo); 165 glCreateBuffers(1, &f->ssbo); 166 glNamedBufferStorage(f->ssbo, f->length * (i32)sizeof(f32) * (fp.complex? 2 : 1), filter, 0); 167 glObjectLabel(GL_BUFFER, f->ssbo, (i32)label.len, (c8 *)label.data); 168 } 169 170 function ComputeFrameIterator 171 compute_frame_iterator(BeamformerCtx *ctx, u32 start_index, u32 needed_frames) 172 { 173 start_index = start_index % ARRAY_COUNT(ctx->beamform_frames); 174 175 ComputeFrameIterator result; 176 result.frames = ctx->beamform_frames; 177 result.offset = start_index; 178 result.capacity = ARRAY_COUNT(ctx->beamform_frames); 179 result.cursor = 0; 180 result.needed_frames = needed_frames; 181 return result; 182 } 183 184 function BeamformerFrame * 185 frame_next(ComputeFrameIterator *bfi) 186 { 187 BeamformerFrame *result = 0; 188 if (bfi->cursor != bfi->needed_frames) { 189 u32 index = (bfi->offset + bfi->cursor++) % bfi->capacity; 190 result = bfi->frames + index; 191 } 192 return result; 193 } 194 195 function b32 196 beamformer_frame_compatible(BeamformerFrame *f, iv3 dim, GLenum gl_kind) 197 { 198 b32 result = gl_kind == f->gl_kind && iv3_equal(dim, f->dim); 199 return result; 200 } 201 202 function iv3 203 make_valid_output_points(i32 points[3]) 204 { 205 iv3 result; 206 result.E[0] = MAX(1, points[0]); 207 result.E[1] = MAX(1, points[1]); 208 result.E[2] = MAX(1, points[2]); 209 return result; 210 } 211 212 function void 213 alloc_beamform_frame(GLParams *gp, BeamformerFrame *out, iv3 out_dim, GLenum gl_kind, s8 name, Arena arena) 214 { 215 out->dim = make_valid_output_points(out_dim.E); 216 if (gp) { 217 out->dim.x = MIN(out->dim.x, gp->max_3d_texture_dim); 218 out->dim.y = MIN(out->dim.y, gp->max_3d_texture_dim); 219 out->dim.z = MIN(out->dim.z, gp->max_3d_texture_dim); 220 } 221 222 /* NOTE: allocate storage for beamformed output data; 223 * this is shared between compute and fragment shaders */ 224 u32 max_dim = (u32)MAX(out->dim.x, MAX(out->dim.y, out->dim.z)); 225 out->mips = (i32)ctz_u32(round_up_power_of_2(max_dim)) + 1; 226 227 out->gl_kind = gl_kind; 228 229 Stream label = arena_stream(arena); 230 stream_append_s8(&label, name); 231 stream_append_byte(&label, '['); 232 stream_append_hex_u64(&label, out->id); 233 stream_append_byte(&label, ']'); 234 235 glDeleteTextures(1, &out->texture); 236 glCreateTextures(GL_TEXTURE_3D, 1, &out->texture); 237 glTextureStorage3D(out->texture, out->mips, gl_kind, out->dim.x, out->dim.y, out->dim.z); 238 239 glTextureParameteri(out->texture, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 240 glTextureParameteri(out->texture, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 241 242 LABEL_GL_OBJECT(GL_TEXTURE, out->texture, stream_to_s8(&label)); 243 } 244 245 function void 246 update_hadamard_texture(BeamformerComputePlan *cp, i32 order, Arena arena) 247 { 248 f32 *hadamard = make_hadamard_transpose(&arena, order); 249 if (hadamard) { 250 cp->hadamard_order = order; 251 u32 *texture = cp->textures + BeamformerComputeTextureKind_Hadamard; 252 glDeleteTextures(1, texture); 253 glCreateTextures(GL_TEXTURE_2D, 1, texture); 254 glTextureStorage2D(*texture, 1, GL_R32F, order, order); 255 glTextureSubImage2D(*texture, 0, 0, 0, order, order, GL_RED, GL_FLOAT, hadamard); 256 257 Stream label = arena_stream(arena); 258 stream_append_s8(&label, s8("Hadamard")); 259 stream_append_i64(&label, order); 260 LABEL_GL_OBJECT(GL_TEXTURE, *texture, stream_to_s8(&label)); 261 } 262 } 263 264 function void 265 alloc_shader_storage(BeamformerCtx *ctx, u32 decoded_data_size, Arena arena) 266 { 267 BeamformerComputeContext *cc = &ctx->compute_context; 268 glDeleteBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos); 269 glCreateBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos); 270 271 cc->ping_pong_ssbo_size = decoded_data_size; 272 273 Stream label = arena_stream(arena); 274 stream_append_s8(&label, s8("PingPongSSBO[")); 275 i32 s_widx = label.widx; 276 for (i32 i = 0; i < countof(cc->ping_pong_ssbos); i++) { 277 glNamedBufferStorage(cc->ping_pong_ssbos[i], (iz)decoded_data_size, 0, 0); 278 stream_append_i64(&label, i); 279 stream_append_byte(&label, ']'); 280 LABEL_GL_OBJECT(GL_BUFFER, cc->ping_pong_ssbos[i], stream_to_s8(&label)); 281 stream_reset(&label, s_widx); 282 } 283 284 /* TODO(rnp): (25.08.04) cuda lib is heavily broken atm. First there are multiple RF 285 * buffers and cuda decode shouldn't assume that the data is coming from the rf_buffer 286 * ssbo. Second each parameter block may need a different hadamard matrix so ideally 287 * decode should just take the texture as a parameter. Third, none of these dimensions 288 * need to be pre-known by the library unless its allocating GPU memory which it shouldn't 289 * need to do. For now grab out of parameter block 0 but it is not correct */ 290 BeamformerParameterBlock *pb = beamformer_parameter_block(ctx->shared_memory.region, 0); 291 /* NOTE(rnp): these are stubs when CUDA isn't supported */ 292 cuda_register_buffers(cc->ping_pong_ssbos, countof(cc->ping_pong_ssbos), cc->rf_buffer.ssbo); 293 u32 decoded_data_dimension[3] = {pb->parameters.sample_count, pb->parameters.channel_count, pb->parameters.acquisition_count}; 294 cuda_init(pb->parameters.raw_data_dimensions, decoded_data_dimension); 295 } 296 297 function void 298 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info) 299 { 300 u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer); 301 t->buffer[index] = info; 302 } 303 304 function b32 305 fill_frame_compute_work(BeamformerCtx *ctx, BeamformWork *work, BeamformerViewPlaneTag plane, 306 u32 parameter_block, b32 indirect) 307 { 308 b32 result = work != 0; 309 if (result) { 310 u32 frame_id = atomic_add_u32(&ctx->next_render_frame_index, 1); 311 u32 frame_index = frame_id % countof(ctx->beamform_frames); 312 work->kind = indirect? BeamformerWorkKind_ComputeIndirect : BeamformerWorkKind_Compute; 313 work->lock = BeamformerSharedMemoryLockKind_DispatchCompute; 314 work->compute_context.parameter_block = parameter_block; 315 work->compute_context.frame = ctx->beamform_frames + frame_index; 316 work->compute_context.frame->ready_to_present = 0; 317 work->compute_context.frame->view_plane_tag = plane; 318 work->compute_context.frame->id = frame_id; 319 } 320 return result; 321 } 322 323 function void 324 do_sum_shader(BeamformerComputeContext *cc, u32 *in_textures, u32 in_texture_count, 325 u32 out_texture, iv3 out_data_dim) 326 { 327 /* NOTE: zero output before summing */ 328 glClearTexImage(out_texture, 0, GL_RED, GL_FLOAT, 0); 329 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 330 331 glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F); 332 for (u32 i = 0; i < in_texture_count; i++) { 333 glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 334 glDispatchCompute(ORONE((u32)out_data_dim.x / 32u), 335 ORONE((u32)out_data_dim.y), 336 ORONE((u32)out_data_dim.z / 32u)); 337 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 338 } 339 } 340 341 struct compute_cursor { 342 iv3 cursor; 343 uv3 dispatch; 344 iv3 target; 345 u32 points_per_dispatch; 346 u32 completed_points; 347 u32 total_points; 348 }; 349 350 function struct compute_cursor 351 start_compute_cursor(iv3 dim, u32 max_points) 352 { 353 struct compute_cursor result = {0}; 354 u32 invocations_per_dispatch = DAS_LOCAL_SIZE_X * DAS_LOCAL_SIZE_Y * DAS_LOCAL_SIZE_Z; 355 356 result.dispatch.y = MIN(max_points / invocations_per_dispatch, (u32)ceil_f32((f32)dim.y / DAS_LOCAL_SIZE_Y)); 357 358 u32 remaining = max_points / result.dispatch.y; 359 result.dispatch.x = MIN(remaining / invocations_per_dispatch, (u32)ceil_f32((f32)dim.x / DAS_LOCAL_SIZE_X)); 360 result.dispatch.z = MIN(remaining / (invocations_per_dispatch * result.dispatch.x), 361 (u32)ceil_f32((f32)dim.z / DAS_LOCAL_SIZE_Z)); 362 363 result.target.x = MAX(dim.x / (i32)result.dispatch.x / DAS_LOCAL_SIZE_X, 1); 364 result.target.y = MAX(dim.y / (i32)result.dispatch.y / DAS_LOCAL_SIZE_Y, 1); 365 result.target.z = MAX(dim.z / (i32)result.dispatch.z / DAS_LOCAL_SIZE_Z, 1); 366 367 result.points_per_dispatch = 1; 368 result.points_per_dispatch *= result.dispatch.x * DAS_LOCAL_SIZE_X; 369 result.points_per_dispatch *= result.dispatch.y * DAS_LOCAL_SIZE_Y; 370 result.points_per_dispatch *= result.dispatch.z * DAS_LOCAL_SIZE_Z; 371 372 result.total_points = (u32)(dim.x * dim.y * dim.z); 373 374 return result; 375 } 376 377 function iv3 378 step_compute_cursor(struct compute_cursor *cursor) 379 { 380 cursor->cursor.x += 1; 381 if (cursor->cursor.x >= cursor->target.x) { 382 cursor->cursor.x = 0; 383 cursor->cursor.y += 1; 384 if (cursor->cursor.y >= cursor->target.y) { 385 cursor->cursor.y = 0; 386 cursor->cursor.z += 1; 387 } 388 } 389 390 cursor->completed_points += cursor->points_per_dispatch; 391 392 iv3 result = cursor->cursor; 393 result.x *= (i32)cursor->dispatch.x * DAS_LOCAL_SIZE_X; 394 result.y *= (i32)cursor->dispatch.y * DAS_LOCAL_SIZE_Y; 395 result.z *= (i32)cursor->dispatch.z * DAS_LOCAL_SIZE_Z; 396 397 return result; 398 } 399 400 function b32 401 compute_cursor_finished(struct compute_cursor *cursor) 402 { 403 b32 result = cursor->completed_points >= cursor->total_points; 404 return result; 405 } 406 407 function m4 408 das_voxel_transform_matrix(BeamformerParameters *bp) 409 { 410 v3 min = v3_from_f32_array(bp->output_min_coordinate); 411 v3 max = v3_from_f32_array(bp->output_max_coordinate); 412 v3 extent = v3_abs(v3_sub(max, min)); 413 v3 points = v3_from_iv3(make_valid_output_points(bp->output_points)); 414 415 m4 T1 = m4_translation(v3_scale(v3_sub(points, (v3){{1.0f, 1.0f, 1.0f}}), -0.5f)); 416 m4 T2 = m4_translation(v3_add(min, v3_scale(extent, 0.5f))); 417 m4 S = m4_scale(v3_div(extent, points)); 418 419 m4 R; 420 switch (bp->das_shader_id) { 421 case BeamformerAcquisitionKind_FORCES: 422 case BeamformerAcquisitionKind_UFORCES: 423 case BeamformerAcquisitionKind_Flash: 424 { 425 R = m4_identity(); 426 S.c[1].E[1] = 0; 427 T2.c[3].E[1] = 0; 428 }break; 429 case BeamformerAcquisitionKind_HERO_PA: 430 case BeamformerAcquisitionKind_HERCULES: 431 case BeamformerAcquisitionKind_UHERCULES: 432 case BeamformerAcquisitionKind_RCA_TPW: 433 case BeamformerAcquisitionKind_RCA_VLS: 434 { 435 R = m4_rotation_about_z(bp->beamform_plane ? 0.0f : 0.25f); 436 if (!(points.x > 1 && points.y > 1 && points.z > 1)) 437 T2.c[3].E[1] = bp->off_axis_pos; 438 }break; 439 default:{ R = m4_identity(); }break; 440 } 441 m4 result = m4_mul(R, m4_mul(T2, m4_mul(S, T1))); 442 return result; 443 } 444 445 function void 446 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb) 447 { 448 b32 run_cuda_hilbert = 0; 449 b32 demodulate = 0; 450 451 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 452 switch (pb->pipeline.shaders[i]) { 453 case BeamformerShaderKind_CudaHilbert:{ run_cuda_hilbert = 1; }break; 454 case BeamformerShaderKind_Demodulate:{ demodulate = 1; }break; 455 default:{}break; 456 } 457 } 458 459 if (demodulate) run_cuda_hilbert = 0; 460 461 cp->iq_pipeline = demodulate || run_cuda_hilbert; 462 463 f32 sampling_frequency = pb->parameters.sampling_frequency; 464 u32 decimation_rate = MAX(pb->parameters.decimation_rate, 1); 465 u32 sample_count = pb->parameters.sample_count; 466 if (demodulate) { 467 sample_count /= (2 * decimation_rate); 468 sampling_frequency /= 2 * (f32)decimation_rate; 469 } 470 471 cp->rf_size = sample_count * pb->parameters.channel_count * pb->parameters.acquisition_count; 472 if (cp->iq_pipeline) cp->rf_size *= 8; 473 else cp->rf_size *= 4; 474 475 u32 das_sample_stride = 1; 476 u32 das_transmit_stride = sample_count; 477 u32 das_channel_stride = sample_count * pb->parameters.acquisition_count; 478 479 f32 time_offset = pb->parameters.time_offset; 480 481 BeamformerDataKind data_kind = pb->pipeline.data_kind; 482 cp->pipeline.shader_count = 0; 483 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 484 BeamformerShaderParameters *sp = pb->pipeline.parameters + i; 485 u32 slot = cp->pipeline.shader_count; 486 u32 shader = pb->pipeline.shaders[i]; 487 b32 commit = 0; 488 489 BeamformerShaderDescriptor *ld = cp->shader_descriptors + slot - 1; 490 BeamformerShaderDescriptor *sd = cp->shader_descriptors + slot; 491 zero_struct(sd); 492 493 switch (shader) { 494 case BeamformerShaderKind_CudaHilbert:{ commit = run_cuda_hilbert; }break; 495 case BeamformerShaderKind_Decode:{ 496 /* TODO(rnp): rework decode first and demodulate after */ 497 b32 first = slot == 0; 498 499 sd->bake.data_kind = data_kind; 500 if (!first) { 501 if (data_kind == BeamformerDataKind_Int16) { 502 sd->bake.data_kind = BeamformerDataKind_Int16Complex; 503 } else { 504 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 505 } 506 } 507 508 if (run_cuda_hilbert) sd->bake.flags |= BeamformerShaderDecodeFlags_DilateOutput; 509 510 BeamformerShaderKind *last_shader = cp->pipeline.shaders + slot - 1; 511 assert(first || ((*last_shader == BeamformerShaderKind_Demodulate || 512 *last_shader == BeamformerShaderKind_Filter))); 513 514 BeamformerShaderDecodeBakeParameters *db = &sd->bake.Decode; 515 db->decode_mode = pb->parameters.decode_mode; 516 db->transmit_count = pb->parameters.acquisition_count; 517 518 db->input_sample_stride = first? 1 : ld->bake.Filter.output_sample_stride; 519 db->input_channel_stride = first? pb->parameters.raw_data_dimensions[0] : ld->bake.Filter.output_channel_stride; 520 db->input_transmit_stride = first? pb->parameters.sample_count : 1; 521 522 db->output_sample_stride = das_sample_stride; 523 db->output_channel_stride = das_channel_stride; 524 db->output_transmit_stride = das_transmit_stride; 525 if (first) { 526 db->output_channel_stride *= decimation_rate; 527 db->output_transmit_stride *= decimation_rate; 528 } 529 530 db->transmits_processed = db->transmit_count >= 32 ? 2 : 1; 531 532 b32 use_16z = db->transmit_count <= 32 || db->transmit_count == 80 || db->transmit_count == 96 || db->transmit_count == 160; 533 sd->layout.x = 4; 534 sd->layout.y = 1; 535 sd->layout.z = use_16z? 16 : 32; 536 537 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 538 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 539 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z / (f32)db->transmits_processed); 540 541 if (first) sd->dispatch.x *= decimation_rate; 542 543 /* NOTE(rnp): decode 2 samples per dispatch when data is i16 */ 544 if (first && data_kind == BeamformerDataKind_Int16) 545 sd->dispatch.x = (u32)ceil_f32((f32)sd->dispatch.x / 2); 546 547 commit = 1; 548 }break; 549 case BeamformerShaderKind_Demodulate: 550 case BeamformerShaderKind_Filter: 551 { 552 b32 first = slot == 0; 553 b32 demod = shader == BeamformerShaderKind_Demodulate; 554 BeamformerFilter *f = cp->filters + sp->filter_slot; 555 556 time_offset += f->time_delay; 557 558 BeamformerShaderFilterBakeParameters *fb = &sd->bake.Filter; 559 fb->filter_length = (u32)f->length; 560 if (demod) sd->bake.flags |= BeamformerShaderFilterFlags_Demodulate; 561 if (f->parameters.complex) sd->bake.flags |= BeamformerShaderFilterFlags_ComplexFilter; 562 if (first) sd->bake.flags |= BeamformerShaderFilterFlags_MapChannels; 563 564 sd->bake.data_kind = data_kind; 565 if (!first) sd->bake.data_kind = BeamformerDataKind_Float32; 566 567 /* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating 568 * between sampling the I portion and the Q portion of an IQ signal. Therefore there 569 * is an implicit decimation factor of 2 which must always be included. All code here 570 * assumes that the signal was sampled in such a way that supports this operation. 571 * To recover IQ[n] from the sampled data (RF[n]) we do the following: 572 * I[n] = RF[n] 573 * Q[n] = RF[n + 1] 574 * IQ[n] = I[n] - j*Q[n] 575 */ 576 if (demod) { 577 fb->demodulation_frequency = pb->parameters.demodulation_frequency; 578 fb->sampling_frequency = pb->parameters.sampling_frequency / 2; 579 fb->decimation_rate = decimation_rate; 580 fb->sample_count = pb->parameters.sample_count; 581 582 if (first) { 583 fb->input_channel_stride = pb->parameters.raw_data_dimensions[0] / 2; 584 fb->input_sample_stride = 1; 585 fb->input_transmit_stride = pb->parameters.sample_count / 2; 586 587 /* NOTE(rnp): output optimized layout for decoding */ 588 fb->output_channel_stride = das_channel_stride; 589 fb->output_sample_stride = pb->parameters.acquisition_count; 590 fb->output_transmit_stride = 1; 591 } else { 592 assert(cp->pipeline.shaders[slot - 1] == BeamformerShaderKind_Decode); 593 fb->input_channel_stride = ld->bake.Decode.output_channel_stride; 594 fb->input_sample_stride = ld->bake.Decode.output_sample_stride; 595 fb->input_transmit_stride = ld->bake.Decode.output_transmit_stride; 596 597 fb->output_channel_stride = das_channel_stride; 598 fb->output_sample_stride = das_sample_stride; 599 fb->output_transmit_stride = das_transmit_stride; 600 } 601 } else { 602 fb->decimation_rate = 1; 603 fb->output_channel_stride = sample_count * pb->parameters.acquisition_count; 604 fb->output_sample_stride = 1; 605 fb->output_transmit_stride = sample_count; 606 fb->input_channel_stride = sample_count * pb->parameters.acquisition_count; 607 fb->input_sample_stride = 1; 608 fb->input_transmit_stride = sample_count; 609 fb->sample_count = sample_count; 610 } 611 612 /* TODO(rnp): filter may need a different dispatch layout */ 613 sd->layout.x = 128; 614 sd->layout.y = 1; 615 sd->layout.z = 1; 616 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 617 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 618 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z); 619 620 commit = 1; 621 }break; 622 case BeamformerShaderKind_DAS:{ 623 sd->bake.data_kind = BeamformerDataKind_Float32; 624 if (cp->iq_pipeline) 625 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 626 627 BeamformerShaderDASBakeParameters *db = &sd->bake.DAS; 628 BeamformerDASUBO *du = &cp->das_ubo_data; 629 du->voxel_transform = das_voxel_transform_matrix(&pb->parameters); 630 mem_copy(du->xdc_transform.E, pb->parameters.xdc_transform, sizeof(du->xdc_transform)); 631 mem_copy(du->xdc_element_pitch.E, pb->parameters.xdc_element_pitch, sizeof(du->xdc_element_pitch)); 632 db->sampling_frequency = sampling_frequency; 633 db->demodulation_frequency = pb->parameters.demodulation_frequency; 634 db->speed_of_sound = pb->parameters.speed_of_sound; 635 db->time_offset = time_offset; 636 db->f_number = pb->parameters.f_number; 637 db->acquisition_kind = pb->parameters.das_shader_id; 638 db->sample_count = sample_count; 639 db->channel_count = pb->parameters.channel_count; 640 db->acquisition_count = pb->parameters.acquisition_count; 641 db->interpolation_mode = pb->parameters.interpolation_mode; 642 db->transmit_angle = pb->parameters.focal_vector[0]; 643 db->focus_depth = pb->parameters.focal_vector[1]; 644 db->transmit_receive_orientation = pb->parameters.transmit_receive_orientation; 645 646 if (pb->parameters.single_focus) sd->bake.flags |= BeamformerShaderDASFlags_SingleFocus; 647 if (pb->parameters.single_orientation) sd->bake.flags |= BeamformerShaderDASFlags_SingleOrientation; 648 if (pb->parameters.coherency_weighting) sd->bake.flags |= BeamformerShaderDASFlags_CoherencyWeighting; 649 else sd->bake.flags |= BeamformerShaderDASFlags_Fast; 650 651 u32 id = pb->parameters.das_shader_id; 652 if (id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_UHERCULES) 653 sd->bake.flags |= BeamformerShaderDASFlags_Sparse; 654 655 sd->layout.x = DAS_LOCAL_SIZE_X; 656 sd->layout.y = DAS_LOCAL_SIZE_Y; 657 sd->layout.z = DAS_LOCAL_SIZE_Z; 658 659 commit = 1; 660 }break; 661 default:{ commit = 1; }break; 662 } 663 664 if (commit) { 665 u32 index = cp->pipeline.shader_count++; 666 cp->pipeline.shaders[index] = shader; 667 cp->pipeline.parameters[index] = *sp; 668 } 669 } 670 cp->pipeline.data_kind = data_kind; 671 } 672 673 function void 674 stream_push_shader_header(Stream *s, BeamformerShaderKind shader_kind, s8 header) 675 { 676 stream_append_s8s(s, s8("#version 460 core\n\n"), header); 677 678 switch (shader_kind) { 679 case BeamformerShaderKind_DAS:{ 680 stream_append_s8(s, s8("" 681 "layout(location = " str(DAS_VOXEL_OFFSET_UNIFORM_LOC) ") uniform ivec3 u_voxel_offset;\n" 682 "layout(location = " str(DAS_CYCLE_T_UNIFORM_LOC) ") uniform uint u_cycle_t;\n" 683 "layout(location = " str(DAS_FAST_CHANNEL_UNIFORM_LOC) ") uniform int u_channel;\n\n" 684 )); 685 }break; 686 case BeamformerShaderKind_Decode:{ 687 stream_append_s8s(s, s8("" 688 "layout(location = " str(DECODE_FIRST_PASS_UNIFORM_LOC) ") uniform bool u_first_pass;\n\n" 689 )); 690 }break; 691 case BeamformerShaderKind_MinMax:{ 692 stream_append_s8(s, s8("layout(location = " str(MIN_MAX_MIPS_LEVEL_UNIFORM_LOC) 693 ") uniform int u_mip_map;\n\n")); 694 }break; 695 case BeamformerShaderKind_Sum:{ 696 stream_append_s8(s, s8("layout(location = " str(SUM_PRESCALE_UNIFORM_LOC) 697 ") uniform float u_sum_prescale = 1.0;\n\n")); 698 }break; 699 default:{}break; 700 } 701 } 702 703 function void 704 load_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 shader_slot, Arena arena) 705 { 706 read_only local_persist s8 compute_headers[BeamformerShaderKind_ComputeCount] = { 707 /* X(name, type, gltype) */ 708 #define X(name, t, gltype) "\t" #gltype " " #name ";\n" 709 [BeamformerShaderKind_DAS] = s8_comp("layout(std140, binding = 0) uniform parameters {\n" 710 BEAMFORMER_DAS_UBO_PARAM_LIST 711 "};\n\n" 712 ), 713 #undef X 714 }; 715 716 BeamformerShaderKind shader = cp->pipeline.shaders[shader_slot]; 717 718 u32 program = 0; 719 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[shader]; 720 if (reloadable_index != -1) { 721 BeamformerShaderKind base_shader = beamformer_reloadable_shader_kinds[reloadable_index]; 722 s8 path; 723 if (!BakeShaders) 724 path = push_s8_from_parts(&arena, ctx->os.path_separator, s8("shaders"), 725 beamformer_reloadable_shader_files[reloadable_index]); 726 727 Stream shader_stream = arena_stream(arena); 728 stream_push_shader_header(&shader_stream, base_shader, compute_headers[base_shader]); 729 730 i32 header_vector_length = beamformer_shader_header_vector_lengths[reloadable_index]; 731 i32 *header_vector = beamformer_shader_header_vectors[reloadable_index]; 732 for (i32 index = 0; index < header_vector_length; index++) 733 stream_append_s8(&shader_stream, beamformer_shader_global_header_strings[header_vector[index]]); 734 735 if (beamformer_shader_bake_parameter_counts[reloadable_index]) { 736 i32 count = beamformer_shader_bake_parameter_counts[reloadable_index]; 737 BeamformerShaderDescriptor *sd = cp->shader_descriptors + shader_slot; 738 739 if (sd->layout.x != 0) { 740 stream_append_s8(&shader_stream, s8("layout(local_size_x = ")); 741 stream_append_u64(&shader_stream, sd->layout.x); 742 stream_append_s8(&shader_stream, s8(", local_size_y = ")); 743 stream_append_u64(&shader_stream, sd->layout.y); 744 stream_append_s8(&shader_stream, s8(", local_size_z = ")); 745 stream_append_u64(&shader_stream, sd->layout.z); 746 stream_append_s8(&shader_stream, s8(") in;\n\n")); 747 } 748 749 u32 *parameters = (u32 *)&sd->bake; 750 s8 *names = beamformer_shader_bake_parameter_names[reloadable_index]; 751 u8 *is_float = beamformer_shader_bake_parameter_is_float[reloadable_index]; 752 for (i32 index = 0; index < count; index++) { 753 stream_append_s8s(&shader_stream, s8("#define "), names[index], 754 is_float[index]? s8(" uintBitsToFloat") : s8(" "), s8("(0x")); 755 stream_append_hex_u64(&shader_stream, parameters[index]); 756 stream_append_s8(&shader_stream, s8(")\n")); 757 } 758 759 stream_append_s8(&shader_stream, s8("#define DataKind (0x")); 760 stream_append_hex_u64(&shader_stream, sd->bake.data_kind); 761 stream_append_s8(&shader_stream, s8(")\n\n")); 762 763 s8 *flag_names = beamformer_shader_flag_strings[reloadable_index]; 764 u32 flag_count = beamformer_shader_flag_strings_count[reloadable_index]; 765 u32 flags = sd->bake.flags; 766 for (u32 bit = 0; bit < flag_count; bit++) { 767 stream_append_s8s(&shader_stream, s8("#define "), flag_names[bit], 768 (flags & (1 << bit))? s8(" 1") : s8(" 0"), s8("\n")); 769 } 770 } 771 772 stream_append_s8(&shader_stream, s8("\n#line 1\n")); 773 774 s8 shader_text; 775 if (BakeShaders) { 776 stream_append_s8(&shader_stream, beamformer_shader_data[reloadable_index]); 777 shader_text = arena_stream_commit(&arena, &shader_stream); 778 } else { 779 shader_text = arena_stream_commit(&arena, &shader_stream); 780 s8 file_text = os_read_whole_file(&arena, (c8 *)path.data); 781 782 assert(shader_text.data + shader_text.len == file_text.data); 783 shader_text.len += file_text.len; 784 } 785 786 /* TODO(rnp): instance name */ 787 s8 shader_name = beamformer_shader_names[shader]; 788 program = load_shader(&ctx->os, arena, &shader_text, (u32 []){GL_COMPUTE_SHADER}, 1, shader_name); 789 } 790 791 glDeleteProgram(cp->programs[shader_slot]); 792 cp->programs[shader_slot] = program; 793 } 794 795 function void 796 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena arena) 797 { 798 BeamformerParameterBlock *pb = beamformer_parameter_block_lock(&ctx->shared_memory, block, -1); 799 for EachBit(pb->dirty_regions, region) { 800 switch (region) { 801 case BeamformerParameterBlockRegion_ComputePipeline: 802 case BeamformerParameterBlockRegion_Parameters: 803 { 804 plan_compute_pipeline(cp, pb); 805 806 /* NOTE(rnp): these are both handled by plan_compute_pipeline() */ 807 u32 mask = 1 << BeamformerParameterBlockRegion_ComputePipeline | 808 1 << BeamformerParameterBlockRegion_Parameters; 809 pb->dirty_regions &= ~mask; 810 811 for (u32 shader_slot = 0; shader_slot < cp->pipeline.shader_count; shader_slot++) { 812 u128 hash = u128_hash_from_data(cp->shader_descriptors + shader_slot, sizeof(BeamformerShaderDescriptor)); 813 if (!u128_equal(hash, cp->shader_hashes[shader_slot])) 814 cp->dirty_programs |= 1 << shader_slot; 815 cp->shader_hashes[shader_slot] = hash; 816 } 817 818 #define X(k, t, v) glNamedBufferSubData(cp->ubos[BeamformerComputeUBOKind_##k], \ 819 0, sizeof(t), &cp->v ## _ubo_data); 820 BEAMFORMER_COMPUTE_UBO_LIST 821 #undef X 822 823 cp->acquisition_count = pb->parameters.acquisition_count; 824 cp->acquisition_kind = pb->parameters.das_shader_id; 825 826 u32 decoded_data_size = cp->rf_size; 827 if (ctx->compute_context.ping_pong_ssbo_size < decoded_data_size) 828 alloc_shader_storage(ctx, decoded_data_size, arena); 829 830 if (cp->hadamard_order != (i32)cp->acquisition_count) 831 update_hadamard_texture(cp, (i32)cp->acquisition_count, arena); 832 833 cp->min_coordinate = v3_from_f32_array(pb->parameters.output_min_coordinate); 834 cp->max_coordinate = v3_from_f32_array(pb->parameters.output_max_coordinate); 835 836 cp->output_points = make_valid_output_points(pb->parameters.output_points); 837 cp->average_frames = pb->parameters.output_points[3]; 838 839 GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F; 840 if (cp->average_frames > 1 && !beamformer_frame_compatible(ctx->averaged_frames + 0, cp->output_points, gl_kind)) { 841 alloc_beamform_frame(&ctx->gl, ctx->averaged_frames + 0, cp->output_points, gl_kind, s8("Averaged Frame"), arena); 842 alloc_beamform_frame(&ctx->gl, ctx->averaged_frames + 1, cp->output_points, gl_kind, s8("Averaged Frame"), arena); 843 } 844 }break; 845 case BeamformerParameterBlockRegion_ChannelMapping:{ 846 cuda_set_channel_mapping(pb->channel_mapping); 847 } /* FALLTHROUGH */ 848 case BeamformerParameterBlockRegion_FocalVectors: 849 case BeamformerParameterBlockRegion_SparseElements: 850 case BeamformerParameterBlockRegion_TransmitReceiveOrientations: 851 { 852 BeamformerComputeTextureKind texture_kind = 0; 853 u32 pixel_type = 0, texture_format = 0; 854 switch (region) { 855 #define X(kind, _gl, tf, pt, ...) \ 856 case BeamformerParameterBlockRegion_## kind:{ \ 857 texture_kind = BeamformerComputeTextureKind_## kind; \ 858 texture_format = tf; \ 859 pixel_type = pt; \ 860 }break; 861 BEAMFORMER_COMPUTE_TEXTURE_LIST 862 #undef X 863 InvalidDefaultCase; 864 } 865 glTextureSubImage1D(cp->textures[texture_kind], 0, 0, BeamformerMaxChannelCount, 866 texture_format, pixel_type, 867 (u8 *)pb + BeamformerParameterBlockRegionOffsets[region]); 868 }break; 869 } 870 } 871 beamformer_parameter_block_unlock(&ctx->shared_memory, block); 872 } 873 874 function void 875 do_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, BeamformerFrame *frame, 876 BeamformerShaderKind shader, u32 shader_slot, BeamformerShaderParameters *sp, Arena arena) 877 { 878 BeamformerComputeContext *cc = &ctx->compute_context; 879 880 u32 program = cp->programs[shader_slot]; 881 glUseProgram(program); 882 883 u32 output_ssbo_idx = !cc->last_output_ssbo_index; 884 u32 input_ssbo_idx = cc->last_output_ssbo_index; 885 886 uv3 dispatch = cp->shader_descriptors[shader_slot].dispatch; 887 switch (shader) { 888 case BeamformerShaderKind_Decode:{ 889 glBindImageTexture(0, cp->textures[BeamformerComputeTextureKind_Hadamard], 0, 0, 0, GL_READ_ONLY, GL_R32F); 890 891 if (shader_slot == 0) { 892 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[input_ssbo_idx]); 893 glBindImageTexture(1, cp->textures[BeamformerComputeTextureKind_ChannelMapping], 0, 0, 0, GL_READ_ONLY, GL_R16I); 894 glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 1); 895 896 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 897 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 898 } 899 900 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]); 901 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cc->ping_pong_ssbos[output_ssbo_idx]); 902 903 glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 0); 904 905 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 906 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 907 908 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 909 }break; 910 case BeamformerShaderKind_CudaDecode:{ 911 cuda_decode(0, output_ssbo_idx, 0); 912 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 913 }break; 914 case BeamformerShaderKind_CudaHilbert:{ 915 cuda_hilbert(input_ssbo_idx, output_ssbo_idx); 916 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 917 }break; 918 case BeamformerShaderKind_Filter: 919 case BeamformerShaderKind_Demodulate: 920 { 921 b32 map_channels = (cp->shader_descriptors[shader_slot].bake.flags & BeamformerShaderFilterFlags_MapChannels) != 0; 922 923 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[output_ssbo_idx]); 924 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cp->filters[sp->filter_slot].ssbo); 925 926 if (!map_channels) 927 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]); 928 else 929 glBindImageTexture(1, cp->textures[BeamformerComputeTextureKind_ChannelMapping], 0, 0, 0, GL_READ_ONLY, GL_R16I); 930 931 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 932 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 933 934 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 935 }break; 936 case BeamformerShaderKind_MinMax:{ 937 for (i32 i = 1; i < frame->mips; i++) { 938 glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 939 glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F); 940 glProgramUniform1i(program, MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i); 941 942 u32 width = (u32)frame->dim.x >> i; 943 u32 height = (u32)frame->dim.y >> i; 944 u32 depth = (u32)frame->dim.z >> i; 945 glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32)); 946 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 947 } 948 }break; 949 case BeamformerShaderKind_DAS:{ 950 local_persist u32 das_cycle_t = 0; 951 952 BeamformerShaderBakeParameters *bp = &cp->shader_descriptors[shader_slot].bake; 953 b32 fast = (bp->flags & BeamformerShaderDASFlags_Fast) != 0; 954 b32 sparse = (bp->flags & BeamformerShaderDASFlags_Sparse) != 0; 955 956 if (fast) { 957 glClearTexImage(frame->texture, 0, GL_RED, GL_FLOAT, 0); 958 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 959 glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_READ_WRITE, cp->iq_pipeline ? GL_RG32F : GL_R32F); 960 } else { 961 glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_WRITE_ONLY, cp->iq_pipeline ? GL_RG32F : GL_R32F); 962 } 963 964 u32 sparse_texture = cp->textures[BeamformerComputeTextureKind_SparseElements]; 965 if (!sparse) sparse_texture = 0; 966 967 glBindBufferBase(GL_UNIFORM_BUFFER, 0, cp->ubos[BeamformerComputeUBOKind_DAS]); 968 glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx], 0, cp->rf_size); 969 glBindImageTexture(1, sparse_texture, 0, 0, 0, GL_READ_ONLY, GL_R16I); 970 glBindImageTexture(2, cp->textures[BeamformerComputeTextureKind_FocalVectors], 0, 0, 0, GL_READ_ONLY, GL_RG32F); 971 glBindImageTexture(3, cp->textures[BeamformerComputeTextureKind_TransmitReceiveOrientations], 0, 0, 0, GL_READ_ONLY, GL_R8I); 972 973 glProgramUniform1ui(program, DAS_CYCLE_T_UNIFORM_LOC, das_cycle_t++); 974 975 if (fast) { 976 i32 loop_end; 977 if (bp->DAS.acquisition_kind == BeamformerAcquisitionKind_RCA_VLS || 978 bp->DAS.acquisition_kind == BeamformerAcquisitionKind_RCA_TPW) 979 { 980 /* NOTE(rnp): to avoid repeatedly sampling the whole focal vectors 981 * texture we loop over transmits for VLS/TPW */ 982 loop_end = (i32)bp->DAS.acquisition_count; 983 } else { 984 loop_end = (i32)bp->DAS.channel_count; 985 } 986 f32 percent_per_step = 1.0f / (f32)loop_end; 987 cc->processing_progress = -percent_per_step; 988 for (i32 index = 0; index < loop_end; index++) { 989 cc->processing_progress += percent_per_step; 990 /* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */ 991 glFinish(); 992 glProgramUniform1i(program, DAS_FAST_CHANNEL_UNIFORM_LOC, index); 993 glDispatchCompute((u32)ceil_f32((f32)frame->dim.x / DAS_LOCAL_SIZE_X), 994 (u32)ceil_f32((f32)frame->dim.y / DAS_LOCAL_SIZE_Y), 995 (u32)ceil_f32((f32)frame->dim.z / DAS_LOCAL_SIZE_Z)); 996 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 997 } 998 } else { 999 #if 1 1000 /* TODO(rnp): compute max_points_per_dispatch based on something like a 1001 * transmit_count * channel_count product */ 1002 u32 max_points_per_dispatch = KB(64); 1003 struct compute_cursor cursor = start_compute_cursor(frame->dim, max_points_per_dispatch); 1004 f32 percent_per_step = (f32)cursor.points_per_dispatch / (f32)cursor.total_points; 1005 cc->processing_progress = -percent_per_step; 1006 for (iv3 offset = {0}; 1007 !compute_cursor_finished(&cursor); 1008 offset = step_compute_cursor(&cursor)) 1009 { 1010 cc->processing_progress += percent_per_step; 1011 /* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */ 1012 glFinish(); 1013 glProgramUniform3iv(program, DAS_VOXEL_OFFSET_UNIFORM_LOC, 1, offset.E); 1014 glDispatchCompute(cursor.dispatch.x, cursor.dispatch.y, cursor.dispatch.z); 1015 } 1016 #else 1017 /* NOTE(rnp): use this for testing tiling code. The performance of the above path 1018 * should be the same as this path if everything is working correctly */ 1019 iv3 compute_dim_offset = {0}; 1020 glProgramUniform3iv(program, DAS_VOXEL_OFFSET_UNIFORM_LOC, 1, compute_dim_offset.E); 1021 glDispatchCompute((u32)ceil_f32((f32)dim.x / DAS_LOCAL_SIZE_X), 1022 (u32)ceil_f32((f32)dim.y / DAS_LOCAL_SIZE_Y), 1023 (u32)ceil_f32((f32)dim.z / DAS_LOCAL_SIZE_Z)); 1024 #endif 1025 } 1026 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT|GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1027 }break; 1028 case BeamformerShaderKind_Sum:{ 1029 u32 aframe_index = ctx->averaged_frame_index % ARRAY_COUNT(ctx->averaged_frames); 1030 BeamformerFrame *aframe = ctx->averaged_frames + aframe_index; 1031 aframe->id = ctx->averaged_frame_index; 1032 atomic_store_u32(&aframe->ready_to_present, 0); 1033 /* TODO(rnp): hack we need a better way of specifying which frames to sum; 1034 * this is fine for rolling averaging but what if we want to do something else */ 1035 assert(frame >= ctx->beamform_frames); 1036 assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames)); 1037 u32 base_index = (u32)(frame - ctx->beamform_frames); 1038 u32 to_average = (u32)cp->average_frames; 1039 u32 frame_count = 0; 1040 u32 *in_textures = push_array(&arena, u32, BeamformerMaxSavedFrames); 1041 ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average); 1042 for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi)) 1043 in_textures[frame_count++] = it->texture; 1044 1045 assert(to_average == frame_count); 1046 1047 glProgramUniform1f(program, SUM_PRESCALE_UNIFORM_LOC, 1 / (f32)frame_count); 1048 do_sum_shader(cc, in_textures, frame_count, aframe->texture, aframe->dim); 1049 aframe->min_coordinate = frame->min_coordinate; 1050 aframe->max_coordinate = frame->max_coordinate; 1051 aframe->compound_count = frame->compound_count; 1052 aframe->acquisition_kind = frame->acquisition_kind; 1053 }break; 1054 InvalidDefaultCase; 1055 } 1056 } 1057 1058 function s8 1059 shader_text_with_header(s8 header, s8 filepath, b32 has_file, BeamformerShaderKind shader_kind, Arena *arena) 1060 { 1061 Stream sb = arena_stream(*arena); 1062 stream_push_shader_header(&sb, shader_kind, header); 1063 stream_append_s8(&sb, s8("\n#line 1\n")); 1064 1065 s8 result; 1066 if (BakeShaders) { 1067 /* TODO(rnp): better handling of shaders with no backing file */ 1068 if (has_file) { 1069 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[shader_kind]; 1070 stream_append_s8(&sb, beamformer_shader_data[reloadable_index]); 1071 } 1072 result = arena_stream_commit(arena, &sb); 1073 } else { 1074 result = arena_stream_commit(arena, &sb); 1075 if (has_file) { 1076 s8 file = os_read_whole_file(arena, (c8 *)filepath.data); 1077 assert(file.data == result.data + result.len); 1078 result.len += file.len; 1079 } 1080 } 1081 1082 return result; 1083 } 1084 1085 /* NOTE(rnp): currently this function is only handling rendering shaders. 1086 * look at load_compute_shader for compute shaders */ 1087 DEBUG_EXPORT BEAMFORMER_RELOAD_SHADER_FN(beamformer_reload_shader) 1088 { 1089 BeamformerCtx *ctx = src->beamformer_context; 1090 BeamformerShaderKind kind = beamformer_reloadable_shader_kinds[src->reloadable_info_index]; 1091 assert(kind == BeamformerShaderKind_Render3D); 1092 1093 i32 shader_count = 1; 1094 ShaderReloadContext *link = src->link; 1095 while (link != src) { shader_count++; link = link->link; } 1096 1097 s8 *shader_texts = push_array(&arena, s8, shader_count); 1098 u32 *shader_types = push_array(&arena, u32, shader_count); 1099 1100 i32 index = 0; 1101 do { 1102 b32 has_file = link->reloadable_info_index >= 0; 1103 shader_texts[index] = shader_text_with_header(link->header, path, has_file, kind, &arena); 1104 shader_types[index] = link->gl_type; 1105 index++; 1106 link = link->link; 1107 } while (link != src); 1108 1109 u32 *shader = &ctx->frame_view_render_context.shader; 1110 glDeleteProgram(*shader); 1111 *shader = load_shader(&ctx->os, arena, shader_texts, shader_types, shader_count, shader_name); 1112 ctx->frame_view_render_context.updated = 1; 1113 1114 return 1; 1115 } 1116 1117 function void 1118 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena, iptr gl_context) 1119 { 1120 BeamformerComputeContext *cs = &ctx->compute_context; 1121 BeamformerSharedMemory *sm = ctx->shared_memory.region; 1122 1123 BeamformWork *work = beamform_work_queue_pop(q); 1124 while (work) { 1125 b32 can_commit = 1; 1126 switch (work->kind) { 1127 case BeamformerWorkKind_ReloadShader:{ 1128 u32 reserved_blocks = sm->reserved_parameter_blocks; 1129 for (u32 block = 0; block < reserved_blocks; block++) { 1130 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena); 1131 for (u32 slot = 0; slot < cp->pipeline.shader_count; slot++) { 1132 i32 shader_index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1133 if (beamformer_reloadable_shader_kinds[shader_index] == work->reload_shader) 1134 cp->dirty_programs |= 1 << slot; 1135 } 1136 } 1137 1138 if (ctx->latest_frame && !sm->live_imaging_parameters.active) { 1139 fill_frame_compute_work(ctx, work, ctx->latest_frame->view_plane_tag, 1140 ctx->latest_frame->parameter_block, 0); 1141 can_commit = 0; 1142 } 1143 }break; 1144 case BeamformerWorkKind_ExportBuffer:{ 1145 /* TODO(rnp): better way of handling DispatchCompute barrier */ 1146 post_sync_barrier(&ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute, sm->locks); 1147 os_shared_memory_region_lock(&ctx->shared_memory, sm->locks, (i32)work->lock, (u32)-1); 1148 BeamformerExportContext *ec = &work->export_context; 1149 switch (ec->kind) { 1150 case BeamformerExportKind_BeamformedData:{ 1151 BeamformerFrame *frame = ctx->latest_frame; 1152 if (frame) { 1153 assert(frame->ready_to_present); 1154 u32 texture = frame->texture; 1155 iv3 dim = frame->dim; 1156 u32 out_size = (u32)dim.x * (u32)dim.y * (u32)dim.z * 2 * sizeof(f32); 1157 if (out_size <= ec->size) { 1158 glGetTextureImage(texture, 0, GL_RG, GL_FLOAT, (i32)out_size, 1159 beamformer_shared_memory_scratch_arena(sm).beg); 1160 } 1161 } 1162 }break; 1163 case BeamformerExportKind_Stats:{ 1164 ComputeTimingTable *table = ctx->compute_timing_table; 1165 /* NOTE(rnp): do a little spin to let this finish updating */ 1166 spin_wait(table->write_index != atomic_load_u32(&table->read_index)); 1167 ComputeShaderStats *stats = ctx->compute_shader_stats; 1168 if (sizeof(stats->table) <= ec->size) 1169 mem_copy(beamformer_shared_memory_scratch_arena(sm).beg, &stats->table, sizeof(stats->table)); 1170 }break; 1171 InvalidDefaultCase; 1172 } 1173 os_shared_memory_region_unlock(&ctx->shared_memory, sm->locks, (i32)work->lock); 1174 post_sync_barrier(&ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync, sm->locks); 1175 }break; 1176 case BeamformerWorkKind_CreateFilter:{ 1177 /* TODO(rnp): this should probably get deleted and moved to lazy loading */ 1178 BeamformerCreateFilterContext *fctx = &work->create_filter_context; 1179 u32 block = fctx->parameter_block; 1180 u32 slot = fctx->filter_slot; 1181 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena); 1182 beamformer_filter_update(cp->filters + slot, fctx->kind, fctx->parameters, block, slot, *arena); 1183 }break; 1184 case BeamformerWorkKind_ComputeIndirect:{ 1185 fill_frame_compute_work(ctx, work, work->compute_indirect_context.view_plane, 1186 work->compute_indirect_context.parameter_block, 1); 1187 } /* FALLTHROUGH */ 1188 case BeamformerWorkKind_Compute:{ 1189 DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[0], GL_RG32F, GL_RG, GL_FLOAT, 0);) 1190 DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[1], GL_RG32F, GL_RG, GL_FLOAT, 0);) 1191 DEBUG_DECL(glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);) 1192 1193 push_compute_timing_info(ctx->compute_timing_table, 1194 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin}); 1195 1196 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_block, arena); 1197 if (beamformer_parameter_block_dirty(sm, work->compute_context.parameter_block)) { 1198 u32 block = work->compute_context.parameter_block; 1199 beamformer_commit_parameter_block(ctx, cp, block, *arena); 1200 atomic_store_u32(&ctx->ui_dirty_parameter_blocks, (u32)(ctx->beamform_work_queue != q) << block); 1201 } 1202 1203 post_sync_barrier(&ctx->shared_memory, work->lock, sm->locks); 1204 1205 u32 dirty_programs = atomic_swap_u32(&cp->dirty_programs, 0); 1206 static_assert(ISPOWEROF2(BeamformerMaxComputeShaderStages), 1207 "max compute shader stages must be power of 2"); 1208 assert((dirty_programs & ~((u32)BeamformerMaxComputeShaderStages - 1)) == 0); 1209 for EachBit(dirty_programs, slot) 1210 load_compute_shader(ctx, cp, (u32)slot, *arena); 1211 1212 atomic_store_u32(&cs->processing_compute, 1); 1213 start_renderdoc_capture(gl_context); 1214 1215 BeamformerFrame *frame = work->compute_context.frame; 1216 1217 GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F; 1218 if (!beamformer_frame_compatible(frame, cp->output_points, gl_kind)) 1219 alloc_beamform_frame(&ctx->gl, frame, cp->output_points, gl_kind, s8("Beamformed_Data"), *arena); 1220 1221 frame->min_coordinate = cp->min_coordinate; 1222 frame->max_coordinate = cp->max_coordinate; 1223 frame->acquisition_kind = cp->acquisition_kind; 1224 frame->compound_count = cp->acquisition_count; 1225 1226 BeamformerComputeContext *cc = &ctx->compute_context; 1227 BeamformerComputePipeline *pipeline = &cp->pipeline; 1228 /* NOTE(rnp): first stage requires access to raw data buffer directly so we break 1229 * it out into a separate step. This way data can get released as soon as possible */ 1230 if (pipeline->shader_count > 0) { 1231 BeamformerRFBuffer *rf = &cs->rf_buffer; 1232 u32 slot = rf->compute_index % countof(rf->compute_syncs); 1233 1234 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1235 /* NOTE(rnp): compute indirect is used when uploading data. if compute thread 1236 * preempts upload it must wait for the fence to exist. then it must tell the 1237 * GPU to wait for upload to complete before it can start compute */ 1238 spin_wait(!atomic_load_u64(rf->upload_syncs + slot)); 1239 1240 glWaitSync(rf->upload_syncs[slot], 0, GL_TIMEOUT_IGNORED); 1241 glDeleteSync(rf->upload_syncs[slot]); 1242 rf->compute_index++; 1243 } else { 1244 slot = (rf->compute_index - 1) % countof(rf->compute_syncs); 1245 } 1246 1247 glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, rf->ssbo, slot * rf->active_rf_size, rf->active_rf_size); 1248 1249 glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[0]); 1250 do_compute_shader(ctx, cp, frame, pipeline->shaders[0], 0, pipeline->parameters + 0, *arena); 1251 glEndQuery(GL_TIME_ELAPSED); 1252 1253 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1254 atomic_store_u64(rf->compute_syncs + slot, glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)); 1255 atomic_store_u64(rf->upload_syncs + slot, 0); 1256 } 1257 } 1258 1259 b32 did_sum_shader = 0; 1260 for (u32 i = 1; i < pipeline->shader_count; i++) { 1261 did_sum_shader |= pipeline->shaders[i] == BeamformerShaderKind_Sum; 1262 glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[i]); 1263 do_compute_shader(ctx, cp, frame, pipeline->shaders[i], i, pipeline->parameters + i, *arena); 1264 glEndQuery(GL_TIME_ELAPSED); 1265 } 1266 1267 /* NOTE(rnp): the first of these blocks until work completes */ 1268 for (u32 i = 0; i < pipeline->shader_count; i++) { 1269 ComputeTimingInfo info = {0}; 1270 info.kind = ComputeTimingInfoKind_Shader; 1271 info.shader = pipeline->shaders[i]; 1272 glGetQueryObjectui64v(cc->shader_timer_ids[i], GL_QUERY_RESULT, &info.timer_count); 1273 push_compute_timing_info(ctx->compute_timing_table, info); 1274 } 1275 cs->processing_progress = 1; 1276 1277 frame->ready_to_present = 1; 1278 if (did_sum_shader) { 1279 u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames)); 1280 ctx->averaged_frames[aframe_index].view_plane_tag = frame->view_plane_tag; 1281 ctx->averaged_frames[aframe_index].ready_to_present = 1; 1282 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index)); 1283 } else { 1284 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame); 1285 } 1286 cs->processing_compute = 0; 1287 1288 push_compute_timing_info(ctx->compute_timing_table, 1289 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd}); 1290 1291 end_renderdoc_capture(gl_context); 1292 }break; 1293 InvalidDefaultCase; 1294 } 1295 1296 if (can_commit) { 1297 beamform_work_queue_pop_commit(q); 1298 work = beamform_work_queue_pop(q); 1299 } 1300 } 1301 } 1302 1303 function void 1304 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats) 1305 { 1306 /* TODO(rnp): we do not currently do anything to handle the potential for a half written 1307 * info item. this could result in garbage entries but they shouldn't really matter */ 1308 1309 u32 target = atomic_load_u32(&t->write_index); 1310 u32 stats_index = (stats->latest_frame_index + 1) % countof(stats->table.times); 1311 1312 static_assert(BeamformerShaderKind_Count + 1 <= 32, "timing coalescence bitfield test"); 1313 u32 seen_info_test = 0; 1314 1315 while (t->read_index != target) { 1316 ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)]; 1317 switch (info.kind) { 1318 case ComputeTimingInfoKind_ComputeFrameBegin:{ 1319 assert(t->compute_frame_active == 0); 1320 t->compute_frame_active = 1; 1321 /* NOTE(rnp): allow multiple instances of same shader to accumulate */ 1322 mem_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index])); 1323 }break; 1324 case ComputeTimingInfoKind_ComputeFrameEnd:{ 1325 assert(t->compute_frame_active == 1); 1326 t->compute_frame_active = 0; 1327 stats->latest_frame_index = stats_index; 1328 stats_index = (stats_index + 1) % countof(stats->table.times); 1329 }break; 1330 case ComputeTimingInfoKind_Shader:{ 1331 stats->table.times[stats_index][info.shader] += (f32)info.timer_count / 1.0e9f; 1332 seen_info_test |= (1u << info.shader); 1333 }break; 1334 case ComputeTimingInfoKind_RF_Data:{ 1335 stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas); 1336 f32 delta = (f32)(info.timer_count - stats->last_rf_timer_count) / 1.0e9f; 1337 stats->table.rf_time_deltas[stats->latest_rf_index] = delta; 1338 stats->last_rf_timer_count = info.timer_count; 1339 seen_info_test |= (1 << BeamformerShaderKind_Count); 1340 }break; 1341 } 1342 /* NOTE(rnp): do this at the end so that stats table is always in a consistent state */ 1343 atomic_add_u32(&t->read_index, 1); 1344 } 1345 1346 if (seen_info_test) { 1347 for EachEnumValue(BeamformerShaderKind, shader) { 1348 if (seen_info_test & (1 << shader)) { 1349 f32 sum = 0; 1350 for EachElement(stats->table.times, i) 1351 sum += stats->table.times[i][shader]; 1352 stats->average_times[shader] = sum / countof(stats->table.times); 1353 } 1354 } 1355 1356 if (seen_info_test & (1 << BeamformerShaderKind_Count)) { 1357 f32 sum = 0; 1358 for EachElement(stats->table.rf_time_deltas, i) 1359 sum += stats->table.rf_time_deltas[i]; 1360 stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas); 1361 } 1362 } 1363 } 1364 1365 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute) 1366 { 1367 BeamformerCtx *ctx = (BeamformerCtx *)user_context; 1368 BeamformerSharedMemory *sm = ctx->shared_memory.region; 1369 complete_queue(ctx, &sm->external_work_queue, arena, gl_context); 1370 complete_queue(ctx, ctx->beamform_work_queue, arena, gl_context); 1371 } 1372 1373 function void 1374 beamformer_rf_buffer_allocate(BeamformerRFBuffer *rf, u32 rf_size, Arena arena) 1375 { 1376 assert((rf_size % 64) == 0); 1377 glDeleteBuffers(1, &rf->ssbo); 1378 glCreateBuffers(1, &rf->ssbo); 1379 1380 glNamedBufferStorage(rf->ssbo, countof(rf->compute_syncs) * rf_size, 0, 1381 GL_DYNAMIC_STORAGE_BIT|GL_MAP_WRITE_BIT); 1382 LABEL_GL_OBJECT(GL_BUFFER, rf->ssbo, s8("Raw_RF_SSBO")); 1383 rf->size = rf_size; 1384 } 1385 1386 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload) 1387 { 1388 BeamformerSharedMemory *sm = ctx->shared_memory->region; 1389 1390 BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace; 1391 BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF; 1392 u32 scratch_rf_size; 1393 if (atomic_load_u32(sm->locks + upload_lock) && 1394 (scratch_rf_size = atomic_swap_u32(&sm->scratch_rf_size, 0)) && 1395 os_shared_memory_region_lock(ctx->shared_memory, sm->locks, (i32)scratch_lock, (u32)-1)) 1396 { 1397 BeamformerRFBuffer *rf = ctx->rf_buffer; 1398 rf->active_rf_size = (u32)round_up_to(scratch_rf_size, 64); 1399 if (rf->size < rf->active_rf_size) 1400 beamformer_rf_buffer_allocate(rf, rf->active_rf_size, arena); 1401 1402 u32 slot = rf->insertion_index++ % countof(rf->compute_syncs); 1403 1404 /* NOTE(rnp): if the rest of the code is functioning then the first 1405 * time the compute thread processes an upload it must have gone 1406 * through this path. therefore it is safe to spin until it gets processed */ 1407 spin_wait(atomic_load_u64(rf->upload_syncs + slot)); 1408 1409 if (atomic_load_u64(rf->compute_syncs + slot)) { 1410 GLenum sync_result = glClientWaitSync(rf->compute_syncs[slot], 0, 1000000000); 1411 if (sync_result == GL_TIMEOUT_EXPIRED || sync_result == GL_WAIT_FAILED) { 1412 // TODO(rnp): what do? 1413 } 1414 glDeleteSync(rf->compute_syncs[slot]); 1415 } 1416 1417 /* NOTE(rnp): nVidia's drivers really don't play nice with persistant mapping, 1418 * at least when it is a big as this one wants to be. mapping and unmapping the 1419 * desired range each time doesn't seem to introduce any performance hit */ 1420 u32 access = GL_MAP_WRITE_BIT|GL_MAP_FLUSH_EXPLICIT_BIT|GL_MAP_UNSYNCHRONIZED_BIT; 1421 u8 *buffer = glMapNamedBufferRange(rf->ssbo, slot * rf->active_rf_size, (i32)rf->active_rf_size, access); 1422 1423 mem_copy(buffer, beamformer_shared_memory_scratch_arena(sm).beg, rf->active_rf_size); 1424 os_shared_memory_region_unlock(ctx->shared_memory, sm->locks, (i32)scratch_lock); 1425 post_sync_barrier(ctx->shared_memory, upload_lock, sm->locks); 1426 1427 glFlushMappedNamedBufferRange(rf->ssbo, 0, (i32)rf->active_rf_size); 1428 glUnmapNamedBuffer(rf->ssbo); 1429 1430 atomic_store_u64(rf->upload_syncs + slot, glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)); 1431 atomic_store_u64(rf->compute_syncs + slot, 0); 1432 1433 os_wake_waiters(ctx->compute_worker_sync); 1434 1435 ComputeTimingInfo info = {.kind = ComputeTimingInfoKind_RF_Data}; 1436 glGetQueryObjectui64v(rf->data_timestamp_query, GL_QUERY_RESULT, &info.timer_count); 1437 glQueryCounter(rf->data_timestamp_query, GL_TIMESTAMP); 1438 push_compute_timing_info(ctx->compute_timing_table, info); 1439 } 1440 } 1441 1442 #include "ui.c" 1443 1444 DEBUG_EXPORT BEAMFORMER_FRAME_STEP_FN(beamformer_frame_step) 1445 { 1446 dt_for_frame = input->dt; 1447 1448 if (IsWindowResized()) { 1449 ctx->window_size.h = GetScreenHeight(); 1450 ctx->window_size.w = GetScreenWidth(); 1451 } 1452 1453 coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats); 1454 1455 if (input->executable_reloaded) { 1456 ui_init(ctx, ctx->ui_backing_store); 1457 DEBUG_DECL(start_frame_capture = ctx->os.start_frame_capture); 1458 DEBUG_DECL(end_frame_capture = ctx->os.end_frame_capture); 1459 } 1460 1461 BeamformerSharedMemory *sm = ctx->shared_memory.region; 1462 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_UploadRF)) 1463 os_wake_waiters(&ctx->os.upload_worker.sync_variable); 1464 1465 BeamformerFrame *frame = ctx->latest_frame; 1466 BeamformerViewPlaneTag tag = frame? frame->view_plane_tag : 0; 1467 draw_ui(ctx, input, frame, tag); 1468 1469 ctx->frame_view_render_context.updated = 0; 1470 1471 if (WindowShouldClose()) 1472 ctx->should_exit = 1; 1473 } 1474 1475 /* NOTE(rnp): functions defined in these shouldn't be visible to the whole program */ 1476 #if _DEBUG 1477 #if OS_LINUX 1478 #include "os_linux.c" 1479 #elif OS_WINDOWS 1480 #include "os_win32.c" 1481 #endif 1482 #endif