ogl_beamforming

Ultrasound Beamforming Implemented with OpenGL
git clone anongit@rnpnr.xyz:ogl_beamforming.git
Log | Files | Refs | Feed | Submodules | README | LICENSE

beamformer.c (58303B)


      1 /* See LICENSE for license details. */
      2 /* TODO(rnp):
      3  * [ ]: measure performance of doing channel mapping in a separate shader
      4  * [ ]: BeamformWorkQueue -> BeamformerWorkQueue
      5  * [ ]: need to keep track of gpu memory in some way
      6  *      - want to be able to store more than 16 2D frames but limit 3D frames
      7  *      - maybe keep track of how much gpu memory is committed for beamformed images
      8  *        and use that to determine when to loop back over existing textures
      9  *      - to do this maybe use a circular linked list instead of a flat array
     10  *      - then have a way of querying how many frames are available for a specific point count
     11  * [ ]: bug: reinit cuda on hot-reload
     12  */
     13 
     14 #include "beamformer.h"
     15 
     16 global f32 dt_for_frame;
     17 global u32 cycle_t;
     18 
     19 #ifndef _DEBUG
     20 #define start_renderdoc_capture(...)
     21 #define end_renderdoc_capture(...)
     22 #else
     23 global renderdoc_start_frame_capture_fn *start_frame_capture;
     24 global renderdoc_end_frame_capture_fn   *end_frame_capture;
     25 #define start_renderdoc_capture(gl) if (start_frame_capture) start_frame_capture(gl, 0)
     26 #define end_renderdoc_capture(gl)   if (end_frame_capture)   end_frame_capture(gl, 0)
     27 #endif
     28 
     29 typedef struct {
     30 	BeamformerFrame *frames;
     31 	u32 capacity;
     32 	u32 offset;
     33 	u32 cursor;
     34 	u32 needed_frames;
     35 } ComputeFrameIterator;
     36 
     37 function void
     38 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block)
     39 {
     40 	assert(block < countof(cc->compute_plans));
     41 	BeamformerComputePlan *cp = cc->compute_plans[block];
     42 	if (cp) {
     43 		glDeleteBuffers(countof(cp->ubos), cp->ubos);
     44 		glDeleteTextures(countof(cp->textures), cp->textures);
     45 		for (u32 i = 0; i < countof(cp->filters); i++)
     46 			glDeleteTextures(1, &cp->filters[i].texture);
     47 		cc->compute_plans[block] = 0;
     48 		SLLPushFreelist(cp, cc->compute_plan_freelist);
     49 	}
     50 }
     51 
     52 function BeamformerComputePlan *
     53 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena)
     54 {
     55 	assert(block < countof(cc->compute_plans));
     56 	BeamformerComputePlan *result = cc->compute_plans[block];
     57 	if (!result) {
     58 		result = SLLPopFreelist(cc->compute_plan_freelist);
     59 		if (result) zero_struct(result);
     60 		else        result = push_struct(arena, BeamformerComputePlan);
     61 		cc->compute_plans[block] = result;
     62 
     63 		glCreateBuffers(countof(result->ubos), result->ubos);
     64 
     65 		Stream label = arena_stream(*arena);
     66 		#define X(k, t, ...) \
     67 			glNamedBufferStorage(result->ubos[BeamformerComputeUBOKind_##k], sizeof(t), \
     68 			                     0, GL_DYNAMIC_STORAGE_BIT); \
     69 			stream_append_s8(&label, s8(#t "[")); \
     70 			stream_append_u64(&label, block);     \
     71 			stream_append_byte(&label, ']');      \
     72 			glObjectLabel(GL_BUFFER, result->ubos[BeamformerComputeUBOKind_##k], \
     73 			              label.widx, (c8 *)label.data); \
     74 			label.widx = 0;
     75 		BEAMFORMER_COMPUTE_UBO_LIST
     76 		#undef X
     77 
     78 		#define X(_k, t, ...) t,
     79 		GLenum gl_kind[] = {BEAMFORMER_COMPUTE_TEXTURE_LIST};
     80 		#undef X
     81 		read_only local_persist s8 tex_prefix[] = {
     82 			#define X(k, ...) s8_comp(#k "["),
     83 			BEAMFORMER_COMPUTE_TEXTURE_LIST
     84 			#undef X
     85 		};
     86 		glCreateTextures(GL_TEXTURE_1D, BeamformerComputeTextureKind_Count - 1, result->textures);
     87 		for (u32 i = 0; i < BeamformerComputeTextureKind_Count - 1; i++) {
     88 			/* TODO(rnp): this could be predicated on channel count for this compute plan */
     89 			glTextureStorage1D(result->textures[i], 1, gl_kind[i], BeamformerMaxChannelCount);
     90 			stream_append_s8(&label, tex_prefix[i]);
     91 			stream_append_u64(&label, block);
     92 			stream_append_byte(&label, ']');
     93 			glObjectLabel(GL_TEXTURE, result->textures[i], label.widx, (c8 *)label.data);
     94 			label.widx = 0;
     95 		}
     96 	}
     97 	return result;
     98 }
     99 
    100 function void
    101 beamformer_filter_update(BeamformerFilter *f, BeamformerFilterKind kind,
    102                          BeamformerFilterParameters fp, u32 block, u32 slot, Arena arena)
    103 {
    104 	#define X(k, ...) s8_comp(#k "Filter"),
    105 	read_only local_persist s8 filter_kinds[] = {BEAMFORMER_FILTER_KIND_LIST(,)};
    106 	#undef X
    107 
    108 	Stream sb = arena_stream(arena);
    109 	stream_append_s8s(&sb, filter_kinds[kind % countof(filter_kinds)], s8("["));
    110 	stream_append_u64(&sb, block);
    111 	stream_append_s8(&sb, s8("]["));
    112 	stream_append_u64(&sb, slot);
    113 	stream_append_byte(&sb, ']');
    114 	s8 label = arena_stream_commit(&arena, &sb);
    115 
    116 	void *filter = 0;
    117 	switch (kind) {
    118 	case BeamformerFilterKind_Kaiser:{
    119 		/* TODO(rnp): this should also support complex */
    120 		/* TODO(rnp): implement this as an IFIR filter instead to reduce computation */
    121 		filter = kaiser_low_pass_filter(&arena, fp.Kaiser.cutoff_frequency, fp.sampling_frequency,
    122 		                                fp.Kaiser.beta, (i32)fp.Kaiser.length);
    123 		f->length     = (i32)fp.Kaiser.length;
    124 		f->time_delay = (f32)f->length / 2.0f / fp.sampling_frequency;
    125 	}break;
    126 	case BeamformerFilterKind_MatchedChirp:{
    127 		typeof(fp.MatchedChirp) *mc = &fp.MatchedChirp;
    128 		f32 fs    = fp.sampling_frequency;
    129 		f->length = (i32)(mc->duration * fs);
    130 		if (fp.complex) {
    131 			filter = baseband_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1, 0.5f);
    132 			f->time_delay = complex_filter_first_moment(filter, f->length, fs);
    133 		} else {
    134 			filter = rf_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1);
    135 			f->time_delay = real_filter_first_moment(filter, f->length, fs);
    136 		}
    137 	}break;
    138 	InvalidDefaultCase;
    139 	}
    140 
    141 	f->kind       = kind;
    142 	f->parameters = fp;
    143 
    144 	glDeleteTextures(1, &f->texture);
    145 	glCreateTextures(GL_TEXTURE_1D, 1, &f->texture);
    146 	glTextureStorage1D(f->texture, 1, fp.complex? GL_RG32F : GL_R32F, f->length);
    147 	glTextureSubImage1D(f->texture, 0, 0, f->length, fp.complex? GL_RG : GL_RED, GL_FLOAT, filter);
    148 	glObjectLabel(GL_TEXTURE, f->texture, (i32)label.len, (c8 *)label.data);
    149 }
    150 
    151 function ComputeFrameIterator
    152 compute_frame_iterator(BeamformerCtx *ctx, u32 start_index, u32 needed_frames)
    153 {
    154 	start_index = start_index % ARRAY_COUNT(ctx->beamform_frames);
    155 
    156 	ComputeFrameIterator result;
    157 	result.frames        = ctx->beamform_frames;
    158 	result.offset        = start_index;
    159 	result.capacity      = ARRAY_COUNT(ctx->beamform_frames);
    160 	result.cursor        = 0;
    161 	result.needed_frames = needed_frames;
    162 	return result;
    163 }
    164 
    165 function BeamformerFrame *
    166 frame_next(ComputeFrameIterator *bfi)
    167 {
    168 	BeamformerFrame *result = 0;
    169 	if (bfi->cursor != bfi->needed_frames) {
    170 		u32 index = (bfi->offset + bfi->cursor++) % bfi->capacity;
    171 		result    = bfi->frames + index;
    172 	}
    173 	return result;
    174 }
    175 
    176 function b32
    177 beamformer_frame_compatible(BeamformerFrame *f, iv3 dim, GLenum gl_kind)
    178 {
    179 	b32 result = gl_kind == f->gl_kind && iv3_equal(dim, f->dim);
    180 	return result;
    181 }
    182 
    183 function void
    184 alloc_beamform_frame(GLParams *gp, BeamformerFrame *out, iv3 out_dim, GLenum gl_kind, s8 name, Arena arena)
    185 {
    186 	out->dim.x = MAX(1, out_dim.x);
    187 	out->dim.y = MAX(1, out_dim.y);
    188 	out->dim.z = MAX(1, out_dim.z);
    189 
    190 	if (gp) {
    191 		out->dim.x = MIN(out->dim.x, gp->max_3d_texture_dim);
    192 		out->dim.y = MIN(out->dim.y, gp->max_3d_texture_dim);
    193 		out->dim.z = MIN(out->dim.z, gp->max_3d_texture_dim);
    194 	}
    195 
    196 	/* NOTE: allocate storage for beamformed output data;
    197 	 * this is shared between compute and fragment shaders */
    198 	u32 max_dim = (u32)MAX(out->dim.x, MAX(out->dim.y, out->dim.z));
    199 	out->mips   = (i32)ctz_u32(round_up_power_of_2(max_dim)) + 1;
    200 
    201 	out->gl_kind = gl_kind;
    202 
    203 	Stream label = arena_stream(arena);
    204 	stream_append_s8(&label, name);
    205 	stream_append_byte(&label, '[');
    206 	stream_append_hex_u64(&label, out->id);
    207 	stream_append_byte(&label, ']');
    208 
    209 	glDeleteTextures(1, &out->texture);
    210 	glCreateTextures(GL_TEXTURE_3D, 1, &out->texture);
    211 	glTextureStorage3D(out->texture, out->mips, gl_kind, out->dim.x, out->dim.y, out->dim.z);
    212 
    213 	glTextureParameteri(out->texture, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    214 	glTextureParameteri(out->texture, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
    215 
    216 	LABEL_GL_OBJECT(GL_TEXTURE, out->texture, stream_to_s8(&label));
    217 }
    218 
    219 function void
    220 update_hadamard_texture(BeamformerComputePlan *cp, i32 order, Arena arena)
    221 {
    222 	i32 *hadamard = make_hadamard_transpose(&arena, order);
    223 	if (hadamard) {
    224 		cp->hadamard_order = order;
    225 		u32 *texture = cp->textures + BeamformerComputeTextureKind_Hadamard;
    226 		glDeleteTextures(1, texture);
    227 		glCreateTextures(GL_TEXTURE_2D, 1, texture);
    228 		glTextureStorage2D(*texture, 1, GL_R8I, order, order);
    229 		glTextureSubImage2D(*texture, 0, 0, 0,  order, order, GL_RED_INTEGER, GL_INT, hadamard);
    230 
    231 		Stream label = arena_stream(arena);
    232 		stream_append_s8(&label, s8("Hadamard"));
    233 		stream_append_i64(&label, order);
    234 		LABEL_GL_OBJECT(GL_TEXTURE, *texture, stream_to_s8(&label));
    235 	}
    236 }
    237 
    238 function void
    239 alloc_shader_storage(BeamformerCtx *ctx, u32 decoded_data_size, Arena arena)
    240 {
    241 	BeamformerComputeContext *cc = &ctx->compute_context;
    242 	glDeleteBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos);
    243 	glCreateBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos);
    244 
    245 	cc->ping_pong_ssbo_size = decoded_data_size;
    246 
    247 	Stream label = arena_stream(arena);
    248 	stream_append_s8(&label, s8("PingPongSSBO["));
    249 	i32 s_widx = label.widx;
    250 	for (i32 i = 0; i < countof(cc->ping_pong_ssbos); i++) {
    251 		glNamedBufferStorage(cc->ping_pong_ssbos[i], (iz)decoded_data_size, 0, 0);
    252 		stream_append_i64(&label, i);
    253 		stream_append_byte(&label, ']');
    254 		LABEL_GL_OBJECT(GL_BUFFER, cc->ping_pong_ssbos[i], stream_to_s8(&label));
    255 		stream_reset(&label, s_widx);
    256 	}
    257 
    258 	/* TODO(rnp): (25.08.04) cuda lib is heavily broken atm. First there are multiple RF
    259 	 * buffers and cuda decode shouldn't assume that the data is coming from the rf_buffer
    260 	 * ssbo. Second each parameter block may need a different hadamard matrix so ideally
    261 	 * decode should just take the texture as a parameter. Third, none of these dimensions
    262 	 * need to be pre-known by the library unless its allocating GPU memory which it shouldn't
    263 	 * need to do. For now grab out of parameter block 0 but it is not correct */
    264 	BeamformerParameterBlock *pb = beamformer_parameter_block(ctx->shared_memory.region, 0);
    265 	/* NOTE(rnp): these are stubs when CUDA isn't supported */
    266 	cuda_register_buffers(cc->ping_pong_ssbos, countof(cc->ping_pong_ssbos), cc->rf_buffer.ssbo);
    267 	u32 decoded_data_dimension[3] = {pb->parameters.sample_count, pb->parameters.channel_count, pb->parameters.acquisition_count};
    268 	cuda_init(pb->parameters.raw_data_dimensions, decoded_data_dimension);
    269 }
    270 
    271 function void
    272 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info)
    273 {
    274 	u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer);
    275 	t->buffer[index] = info;
    276 }
    277 
    278 function b32
    279 fill_frame_compute_work(BeamformerCtx *ctx, BeamformWork *work, BeamformerViewPlaneTag plane,
    280                         u32 parameter_block, b32 indirect)
    281 {
    282 	b32 result = 0;
    283 	if (work) {
    284 		result = 1;
    285 		u32 frame_id    = atomic_add_u32(&ctx->next_render_frame_index, 1);
    286 		u32 frame_index = frame_id % countof(ctx->beamform_frames);
    287 		work->kind      = indirect? BeamformerWorkKind_ComputeIndirect : BeamformerWorkKind_Compute;
    288 		work->lock      = BeamformerSharedMemoryLockKind_DispatchCompute;
    289 		work->compute_context.parameter_block = parameter_block;
    290 		work->compute_context.frame = ctx->beamform_frames + frame_index;
    291 		work->compute_context.frame->ready_to_present = 0;
    292 		work->compute_context.frame->view_plane_tag   = plane;
    293 		work->compute_context.frame->id               = frame_id;
    294 	}
    295 	return result;
    296 }
    297 
    298 function void
    299 do_sum_shader(BeamformerComputeContext *cc, u32 *in_textures, u32 in_texture_count, f32 in_scale,
    300               u32 out_texture, iv3 out_data_dim)
    301 {
    302 	/* NOTE: zero output before summing */
    303 	glClearTexImage(out_texture, 0, GL_RED, GL_FLOAT, 0);
    304 	glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT);
    305 
    306 	glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F);
    307 	glProgramUniform1f(cc->programs[BeamformerShaderKind_Sum], SUM_PRESCALE_UNIFORM_LOC, in_scale);
    308 	for (u32 i = 0; i < in_texture_count; i++) {
    309 		glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F);
    310 		glDispatchCompute(ORONE((u32)out_data_dim.x / 32u),
    311 		                  ORONE((u32)out_data_dim.y),
    312 		                  ORONE((u32)out_data_dim.z / 32u));
    313 		glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
    314 	}
    315 }
    316 
    317 struct compute_cursor {
    318 	iv3 cursor;
    319 	uv3 dispatch;
    320 	iv3 target;
    321 	u32 points_per_dispatch;
    322 	u32 completed_points;
    323 	u32 total_points;
    324 };
    325 
    326 function struct compute_cursor
    327 start_compute_cursor(iv3 dim, u32 max_points)
    328 {
    329 	struct compute_cursor result = {0};
    330 	u32 invocations_per_dispatch = DAS_LOCAL_SIZE_X * DAS_LOCAL_SIZE_Y * DAS_LOCAL_SIZE_Z;
    331 
    332 	result.dispatch.y = MIN(max_points / invocations_per_dispatch, (u32)ceil_f32((f32)dim.y / DAS_LOCAL_SIZE_Y));
    333 
    334 	u32 remaining     = max_points / result.dispatch.y;
    335 	result.dispatch.x = MIN(remaining / invocations_per_dispatch, (u32)ceil_f32((f32)dim.x / DAS_LOCAL_SIZE_X));
    336 	result.dispatch.z = MIN(remaining / (invocations_per_dispatch * result.dispatch.x),
    337 	                        (u32)ceil_f32((f32)dim.z / DAS_LOCAL_SIZE_Z));
    338 
    339 	result.target.x = MAX(dim.x / (i32)result.dispatch.x / DAS_LOCAL_SIZE_X, 1);
    340 	result.target.y = MAX(dim.y / (i32)result.dispatch.y / DAS_LOCAL_SIZE_Y, 1);
    341 	result.target.z = MAX(dim.z / (i32)result.dispatch.z / DAS_LOCAL_SIZE_Z, 1);
    342 
    343 	result.points_per_dispatch = 1;
    344 	result.points_per_dispatch *= result.dispatch.x * DAS_LOCAL_SIZE_X;
    345 	result.points_per_dispatch *= result.dispatch.y * DAS_LOCAL_SIZE_Y;
    346 	result.points_per_dispatch *= result.dispatch.z * DAS_LOCAL_SIZE_Z;
    347 
    348 	result.total_points = (u32)(dim.x * dim.y * dim.z);
    349 
    350 	return result;
    351 }
    352 
    353 function iv3
    354 step_compute_cursor(struct compute_cursor *cursor)
    355 {
    356 	cursor->cursor.x += 1;
    357 	if (cursor->cursor.x >= cursor->target.x) {
    358 		cursor->cursor.x  = 0;
    359 		cursor->cursor.y += 1;
    360 		if (cursor->cursor.y >= cursor->target.y) {
    361 			cursor->cursor.y  = 0;
    362 			cursor->cursor.z += 1;
    363 		}
    364 	}
    365 
    366 	cursor->completed_points += cursor->points_per_dispatch;
    367 
    368 	iv3 result = cursor->cursor;
    369 	result.x *= (i32)cursor->dispatch.x * DAS_LOCAL_SIZE_X;
    370 	result.y *= (i32)cursor->dispatch.y * DAS_LOCAL_SIZE_Y;
    371 	result.z *= (i32)cursor->dispatch.z * DAS_LOCAL_SIZE_Z;
    372 
    373 	return result;
    374 }
    375 
    376 function b32
    377 compute_cursor_finished(struct compute_cursor *cursor)
    378 {
    379 	b32 result = cursor->completed_points >= cursor->total_points;
    380 	return result;
    381 }
    382 
    383 function m4
    384 das_voxel_transform_matrix(BeamformerParameters *bp)
    385 {
    386 	v3 min = v3_from_f32_array(bp->output_min_coordinate);
    387 	v3 max = v3_from_f32_array(bp->output_max_coordinate);
    388 	v3 extent = v3_abs(v3_sub(max, min));
    389 	v3 points = {{(f32)bp->output_points[0], (f32)bp->output_points[1], (f32)bp->output_points[2]}};
    390 
    391 	m4 T1 = m4_translation(v3_scale(v3_sub(points, (v3){{1.0f, 1.0f, 1.0f}}), -0.5f));
    392 	m4 T2 = m4_translation(v3_add(min, v3_scale(extent, 0.5f)));
    393 	m4 S  = m4_scale(v3_div(extent, points));
    394 
    395 	m4 R;
    396 	switch (bp->das_shader_id) {
    397 	case BeamformerDASKind_FORCES:
    398 	case BeamformerDASKind_UFORCES:
    399 	case BeamformerDASKind_Flash:
    400 	{
    401 		R = m4_identity();
    402 		S.c[1].E[1]  = 0;
    403 		T2.c[3].E[1] = 0;
    404 	}break;
    405 	case BeamformerDASKind_HERCULES:
    406 	case BeamformerDASKind_UHERCULES:
    407 	case BeamformerDASKind_RCA_TPW:
    408 	case BeamformerDASKind_RCA_VLS:
    409 	{
    410 		R = m4_rotation_about_z(bp->beamform_plane ? 0.0f : 0.25f);
    411 		if (!(points.x > 1 && points.y > 1 && points.z > 1))
    412 			T2.c[3].E[1] = bp->off_axis_pos;
    413 	}break;
    414 	default:{ R = m4_identity(); }break;
    415 	}
    416 	m4 result = m4_mul(R, m4_mul(T2, m4_mul(S, T1)));
    417 	return result;
    418 }
    419 
    420 function void
    421 das_ubo_from_beamformer_parameters(BeamformerDASUBO *du, BeamformerParameters *bp)
    422 {
    423 	du->voxel_transform = das_voxel_transform_matrix(bp);
    424 	mem_copy(du->xdc_transform.E,     bp->xdc_transform,     sizeof(du->xdc_transform));
    425 	mem_copy(du->xdc_element_pitch.E, bp->xdc_element_pitch, sizeof(du->xdc_element_pitch));
    426 	du->sampling_frequency     = bp->sampling_frequency;
    427 	du->demodulation_frequency = bp->demodulation_frequency;
    428 	du->speed_of_sound         = bp->speed_of_sound;
    429 	du->time_offset            = bp->time_offset;
    430 	du->f_number               = bp->f_number;
    431 	du->shader_kind            = bp->das_shader_id;
    432 	du->sample_count           = bp->sample_count;
    433 	du->channel_count          = bp->channel_count;
    434 	du->acquisition_count      = bp->acquisition_count;
    435 
    436 	du->shader_flags = 0;
    437 	if (bp->coherency_weighting) du->shader_flags |= BeamformerShaderDASFlags_CoherencyWeighting;
    438 	if (bp->transmit_mode == BeamformerRCAOrientation_Columns)
    439 		du->shader_flags |= BeamformerShaderDASFlags_TxColumns;
    440 	if (bp->receive_mode == BeamformerRCAOrientation_Columns)
    441 		du->shader_flags |= BeamformerShaderDASFlags_RxColumns;
    442 }
    443 
    444 function void
    445 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb)
    446 {
    447 	BeamformerDASUBO *bp = &cp->das_ubo_data;
    448 
    449 	das_ubo_from_beamformer_parameters(bp, &pb->parameters);
    450 
    451 	b32 decode_first = pb->pipeline.shaders[0] == BeamformerShaderKind_Decode;
    452 	b32 run_cuda_hilbert = 0;
    453 	b32 demodulate       = 0;
    454 
    455 	for (u32 i = 0; i < pb->pipeline.shader_count; i++) {
    456 		switch (pb->pipeline.shaders[i]) {
    457 		case BeamformerShaderKind_CudaHilbert:{ run_cuda_hilbert = 1; }break;
    458 		case BeamformerShaderKind_Demodulate:{  demodulate = 1;       }break;
    459 		default:{}break;
    460 		}
    461 	}
    462 
    463 	if (demodulate) run_cuda_hilbert = 0;
    464 
    465 	if (demodulate || run_cuda_hilbert) cp->iq_pipeline = 1;
    466 
    467 	BeamformerDataKind data_kind = pb->pipeline.data_kind;
    468 	cp->pipeline.shader_count = 0;
    469 	for (u32 i = 0; i < pb->pipeline.shader_count; i++) {
    470 		BeamformerShaderParameters *sp = pb->pipeline.parameters + i;
    471 		u32 shader = pb->pipeline.shaders[i];
    472 		b32 commit = 0;
    473 
    474 		iz match = 0;
    475 		switch (shader) {
    476 		case BeamformerShaderKind_CudaHilbert:{ commit = run_cuda_hilbert; }break;
    477 		case BeamformerShaderKind_Decode:{
    478 			/* TODO(rnp): rework decode first and demodulate after */
    479 			BeamformerDataKind decode_data_kind = data_kind;
    480 			if (!decode_first) {
    481 				if (data_kind == BeamformerDataKind_Int16) {
    482 					decode_data_kind = BeamformerDataKind_Int16Complex;
    483 				} else {
    484 					decode_data_kind = BeamformerDataKind_Float32Complex;
    485 				}
    486 			}
    487 			i32 local_flags = 0;
    488 			if (run_cuda_hilbert) local_flags |= BeamformerShaderDecodeFlags_DilateOutput;
    489 			match = beamformer_shader_decode_match(decode_data_kind, local_flags);
    490 			commit = 1;
    491 		}break;
    492 		case BeamformerShaderKind_Demodulate:{
    493 			BeamformerFilter *f = cp->filters + sp->filter_slot;
    494 			i32 local_flags = BeamformerShaderFilterFlags_Demodulate;
    495 			if (f->parameters.complex) local_flags |= BeamformerShaderFilterFlags_ComplexFilter;
    496 			if (!decode_first)         local_flags |= BeamformerShaderFilterFlags_MapChannels;
    497 
    498 			BeamformerDataKind filter_data_kind = data_kind;
    499 			if (decode_first)
    500 				filter_data_kind = BeamformerDataKind_Float32;
    501 
    502 			match = beamformer_shader_demodulate_match(filter_data_kind, pb->parameters.sampling_mode, local_flags);
    503 
    504 			bp->time_offset += f->time_delay;
    505 			commit = 1;
    506 		}break;
    507 		case BeamformerShaderKind_Filter:{
    508 			BeamformerFilter *f = cp->filters + sp->filter_slot;
    509 			i32 local_flags = 0;
    510 			if (f->parameters.complex) local_flags |= BeamformerShaderFilterFlags_ComplexFilter;
    511 
    512 			BeamformerDataKind filter_data_kind = data_kind;
    513 			if (decode_first)
    514 				filter_data_kind = BeamformerDataKind_Float32;
    515 
    516 			match = beamformer_shader_filter_match(filter_data_kind, local_flags);
    517 			bp->time_offset += f->time_delay;
    518 			commit = 1;
    519 		}break;
    520 		case BeamformerShaderKind_DAS:{
    521 			BeamformerDataKind das_data_kind = BeamformerDataKind_Float32;
    522 			if (demodulate || run_cuda_hilbert)
    523 				das_data_kind = BeamformerDataKind_Float32Complex;
    524 
    525 			i32 local_flags = 0;
    526 			if ((bp->shader_flags & BeamformerShaderDASFlags_CoherencyWeighting) == 0)
    527 				local_flags |= BeamformerShaderDASFlags_Fast;
    528 			if (bp->shader_kind == BeamformerDASKind_UFORCES || bp->shader_kind == BeamformerDASKind_UHERCULES)
    529 				local_flags |= BeamformerShaderDASFlags_Sparse;
    530 			if (pb->parameters.interpolate)
    531 				local_flags |= BeamformerShaderDASFlags_Interpolate;
    532 
    533 			match = beamformer_shader_das_match(das_data_kind, local_flags);
    534 			commit = 1;
    535 		}break;
    536 		default:{
    537 			match  = beamformer_shader_descriptors[shader].first_match_vector_index;
    538 			commit = 1;
    539 		}break;
    540 		}
    541 
    542 		if (commit) {
    543 			u32 index = cp->pipeline.shader_count++;
    544 			cp->pipeline.shaders[index]         = shader;
    545 			cp->pipeline.program_indices[index] = (u32)match;
    546 			cp->pipeline.parameters[index]      = *sp;
    547 		}
    548 	}
    549 	cp->pipeline.data_kind = data_kind;
    550 
    551 	u32 das_sample_stride   = 1;
    552 	u32 das_transmit_stride = bp->sample_count;
    553 	u32 das_channel_stride  = bp->acquisition_count * bp->sample_count;
    554 
    555 	u32 decimation_rate = MAX(pb->parameters.decimation_rate, 1);
    556 	if (demodulate) {
    557 		das_channel_stride  /= (2 * decimation_rate);
    558 		das_transmit_stride /= (2 * decimation_rate);
    559 	}
    560 
    561 	u32 input_sample_stride   = 1;
    562 	u32 input_transmit_stride = bp->sample_count;
    563 	u32 input_channel_stride  = pb->parameters.raw_data_dimensions[0];
    564 
    565 	BeamformerDecodeUBO *dp = &cp->decode_ubo_data;
    566 	dp->decode_mode    = pb->parameters.decode;
    567 	dp->transmit_count = bp->acquisition_count;
    568 
    569 	dp->input_sample_stride    = decode_first? input_sample_stride   : bp->acquisition_count;
    570 	dp->input_channel_stride   = decode_first? input_channel_stride  : das_channel_stride;
    571 	dp->input_transmit_stride  = decode_first? input_transmit_stride : 1;
    572 	dp->output_sample_stride   = das_sample_stride;
    573 	dp->output_channel_stride  = das_channel_stride;
    574 	dp->output_transmit_stride = das_transmit_stride;
    575 	if (decode_first) {
    576 		dp->output_channel_stride  *= decimation_rate;
    577 		dp->output_transmit_stride *= decimation_rate;
    578 	}
    579 
    580 	cp->decode_dispatch.x = (u32)ceil_f32((f32)bp->sample_count      / DECODE_LOCAL_SIZE_X);
    581 	cp->decode_dispatch.y = (u32)ceil_f32((f32)bp->channel_count     / DECODE_LOCAL_SIZE_Y);
    582 	cp->decode_dispatch.z = (u32)ceil_f32((f32)bp->acquisition_count / DECODE_LOCAL_SIZE_Z);
    583 
    584 	/* NOTE(rnp): decode 2 samples per dispatch when data is i16 */
    585 	if (decode_first && data_kind == BeamformerDataKind_Int16)
    586 		cp->decode_dispatch.x = (u32)ceil_f32((f32)cp->decode_dispatch.x / 2);
    587 
    588 	/* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating
    589 	 * between sampling the I portion and the Q portion of an IQ signal. Therefore there
    590 	 * is an implicit decimation factor of 2 which must always be included. All code here
    591 	 * assumes that the signal was sampled in such a way that supports this operation.
    592 	 * To recover IQ[n] from the sampled data (RF[n]) we do the following:
    593 	 *   I[n]  = RF[n]
    594 	 *   Q[n]  = RF[n + 1]
    595 	 *   IQ[n] = I[n] - j*Q[n]
    596 	 */
    597 	if (demodulate) {
    598 		BeamformerFilterUBO *mp    = &cp->demod_ubo_data;
    599 		mp->demodulation_frequency = bp->demodulation_frequency;
    600 		mp->sampling_frequency     = bp->sampling_frequency / 2;
    601 		mp->decimation_rate        = decimation_rate;
    602 
    603 		bp->sampling_frequency /= 2 * (f32)mp->decimation_rate;
    604 		bp->sample_count       /= 2 * mp->decimation_rate;
    605 
    606 		if (decode_first) {
    607 			mp->input_channel_stride  = dp->output_channel_stride;
    608 			mp->input_sample_stride   = dp->output_sample_stride;
    609 			mp->input_transmit_stride = dp->output_transmit_stride;
    610 
    611 			mp->output_channel_stride  = das_channel_stride;
    612 			mp->output_sample_stride   = das_sample_stride;
    613 			mp->output_transmit_stride = das_transmit_stride;
    614 		} else {
    615 			mp->input_channel_stride  = input_channel_stride  / 2;
    616 			mp->input_sample_stride   = input_sample_stride;
    617 			mp->input_transmit_stride = input_transmit_stride / 2;
    618 
    619 			/* NOTE(rnp): output optimized layout for decoding */
    620 			mp->output_channel_stride  = dp->input_channel_stride;
    621 			mp->output_sample_stride   = dp->input_sample_stride;
    622 			mp->output_transmit_stride = dp->input_transmit_stride;
    623 
    624 			cp->decode_dispatch.x = (u32)ceil_f32((f32)bp->sample_count / DECODE_LOCAL_SIZE_X);
    625 		}
    626 	}
    627 
    628 	/* TODO(rnp): filter may need a different dispatch layout */
    629 	cp->demod_dispatch.x = (u32)ceil_f32((f32)bp->sample_count      / FILTER_LOCAL_SIZE_X);
    630 	cp->demod_dispatch.y = (u32)ceil_f32((f32)bp->channel_count     / FILTER_LOCAL_SIZE_Y);
    631 	cp->demod_dispatch.z = (u32)ceil_f32((f32)bp->acquisition_count / FILTER_LOCAL_SIZE_Z);
    632 
    633 	cp->rf_size = bp->sample_count * bp->channel_count * bp->acquisition_count;
    634 	if (demodulate || run_cuda_hilbert) cp->rf_size *= 8;
    635 	else                                cp->rf_size *= 4;
    636 
    637 	/* TODO(rnp): UBO per filter stage */
    638 	BeamformerFilterUBO *flt = &cp->filter_ubo_data;
    639 	flt->demodulation_frequency = bp->demodulation_frequency;
    640 	flt->sampling_frequency     = bp->sampling_frequency;
    641 	flt->decimation_rate        = 1;
    642 	flt->output_channel_stride  = bp->sample_count * bp->acquisition_count;
    643 	flt->output_sample_stride   = 1;
    644 	flt->output_transmit_stride = bp->sample_count;
    645 	flt->input_channel_stride   = bp->sample_count * bp->acquisition_count;
    646 	flt->input_sample_stride    = 1;
    647 	flt->input_transmit_stride  = bp->sample_count;
    648 }
    649 
    650 function void
    651 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena arena)
    652 {
    653 	BeamformerParameterBlock *pb = beamformer_parameter_block_lock(&ctx->shared_memory, block, -1);
    654 	for (u32 region = ctz_u32(pb->dirty_regions);
    655 	     region != 32;
    656 	     region = ctz_u32(pb->dirty_regions))
    657 	{
    658 		mark_parameter_block_region_clean(ctx->shared_memory.region, block, region);
    659 		switch (region) {
    660 		case BeamformerParameterBlockRegion_ComputePipeline:
    661 		case BeamformerParameterBlockRegion_Parameters:
    662 		{
    663 			plan_compute_pipeline(cp, pb);
    664 
    665 			/* NOTE(rnp): these are both handled by plan_compute_pipeline() */
    666 			u32 mask = 1 << BeamformerParameterBlockRegion_ComputePipeline |
    667 			           1 << BeamformerParameterBlockRegion_Parameters;
    668 			pb->dirty_regions &= ~mask;
    669 
    670 			#define X(k, t, v) glNamedBufferSubData(cp->ubos[BeamformerComputeUBOKind_##k], \
    671 			                                        0, sizeof(t), &cp->v ## _ubo_data);
    672 			BEAMFORMER_COMPUTE_UBO_LIST
    673 			#undef X
    674 
    675 			u32 decoded_data_size = cp->rf_size;
    676 			if (ctx->compute_context.ping_pong_ssbo_size < decoded_data_size)
    677 				alloc_shader_storage(ctx, decoded_data_size, arena);
    678 
    679 			if (cp->hadamard_order != (i32)cp->das_ubo_data.acquisition_count)
    680 				update_hadamard_texture(cp, (i32)cp->das_ubo_data.acquisition_count, arena);
    681 
    682 			cp->min_coordinate = v3_from_f32_array(pb->parameters.output_min_coordinate);
    683 			cp->max_coordinate = v3_from_f32_array(pb->parameters.output_max_coordinate);
    684 
    685 			cp->output_points.E[0] = MAX(pb->parameters.output_points[0], 1);
    686 			cp->output_points.E[1] = MAX(pb->parameters.output_points[1], 1);
    687 			cp->output_points.E[2] = MAX(pb->parameters.output_points[2], 1);
    688 			cp->average_frames     = pb->parameters.output_points[3];
    689 
    690 			GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F;
    691 			if (cp->average_frames > 1 && !beamformer_frame_compatible(ctx->averaged_frames + 0, cp->output_points, gl_kind)) {
    692 				alloc_beamform_frame(&ctx->gl, ctx->averaged_frames + 0, cp->output_points, gl_kind, s8("Averaged Frame"), arena);
    693 				alloc_beamform_frame(&ctx->gl, ctx->averaged_frames + 1, cp->output_points, gl_kind, s8("Averaged Frame"), arena);
    694 			}
    695 		}break;
    696 		case BeamformerParameterBlockRegion_ChannelMapping:
    697 		case BeamformerParameterBlockRegion_FocalVectors:
    698 		case BeamformerParameterBlockRegion_SparseElements:
    699 		{
    700 			BeamformerComputeTextureKind texture_kind = 0;
    701 			u32 texture_type = 0, texture_format = 0;
    702 			/* TODO(rnp): this whole thing could be a table */
    703 			switch (region) {
    704 			case BeamformerParameterBlockRegion_ChannelMapping:{
    705 				texture_kind   = BeamformerComputeTextureKind_ChannelMapping;
    706 				texture_type   = GL_SHORT;
    707 				texture_format = GL_RED_INTEGER;
    708 				/* TODO(rnp): cuda lib */
    709 				cuda_set_channel_mapping(pb->channel_mapping);
    710 			}break;
    711 			case BeamformerParameterBlockRegion_FocalVectors:{
    712 				texture_kind   = BeamformerComputeTextureKind_FocalVectors;
    713 				texture_type   = GL_FLOAT;
    714 				texture_format = GL_RG;
    715 			}break;
    716 			case BeamformerParameterBlockRegion_SparseElements:{
    717 				texture_kind   = BeamformerComputeTextureKind_SparseElements;
    718 				texture_type   = GL_SHORT;
    719 				texture_format = GL_RED_INTEGER;
    720 			}break;
    721 			InvalidDefaultCase;
    722 			}
    723 			glTextureSubImage1D(cp->textures[texture_kind], 0, 0, BeamformerMaxChannelCount,
    724 			                    texture_format, texture_type,
    725 			                    (u8 *)pb + BeamformerParameterBlockRegionOffsets[region]);
    726 		}break;
    727 		}
    728 	}
    729 	beamformer_parameter_block_unlock(&ctx->shared_memory, block);
    730 }
    731 
    732 function void
    733 do_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, BeamformerFrame *frame,
    734                   BeamformerShaderKind shader, u32 program_index, BeamformerShaderParameters *sp, Arena arena)
    735 {
    736 	BeamformerComputeContext *cc = &ctx->compute_context;
    737 
    738 	i32 *match_vector = beamformer_shader_match_vectors[program_index];
    739 	BeamformerShaderDescriptor *shader_descriptor = beamformer_shader_descriptors + shader;
    740 
    741 	u32 program = cc->programs[program_index];
    742 	glUseProgram(program);
    743 
    744 	u32 output_ssbo_idx = !cc->last_output_ssbo_index;
    745 	u32 input_ssbo_idx  = cc->last_output_ssbo_index;
    746 
    747 	switch (shader) {
    748 	case BeamformerShaderKind_Decode:{
    749 		glBindBufferBase(GL_UNIFORM_BUFFER, 0, cp->ubos[BeamformerComputeUBOKind_Decode]);
    750 		glBindImageTexture(0, cp->textures[BeamformerComputeTextureKind_Hadamard], 0, 0, 0, GL_READ_ONLY, GL_R8I);
    751 
    752 		if (shader == cp->pipeline.shaders[0]) {
    753 			glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[input_ssbo_idx]);
    754 			glBindImageTexture(1, cp->textures[BeamformerComputeTextureKind_ChannelMapping], 0, 0, 0, GL_READ_ONLY, GL_R16I);
    755 			glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 1);
    756 
    757 			glDispatchCompute(cp->decode_dispatch.x, cp->decode_dispatch.y, cp->decode_dispatch.z);
    758 			glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
    759 		}
    760 
    761 		glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]);
    762 		glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cc->ping_pong_ssbos[output_ssbo_idx]);
    763 
    764 		glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 0);
    765 
    766 		glDispatchCompute(cp->decode_dispatch.x, cp->decode_dispatch.y, cp->decode_dispatch.z);
    767 		glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
    768 
    769 		cc->last_output_ssbo_index = !cc->last_output_ssbo_index;
    770 	}break;
    771 	case BeamformerShaderKind_CudaDecode:{
    772 		cuda_decode(0, output_ssbo_idx, 0);
    773 		cc->last_output_ssbo_index = !cc->last_output_ssbo_index;
    774 	}break;
    775 	case BeamformerShaderKind_CudaHilbert:{
    776 		cuda_hilbert(input_ssbo_idx, output_ssbo_idx);
    777 		cc->last_output_ssbo_index = !cc->last_output_ssbo_index;
    778 	}break;
    779 	case BeamformerShaderKind_Filter:
    780 	case BeamformerShaderKind_Demodulate:
    781 	{
    782 		i32 local_flags  = match_vector[shader_descriptor->match_vector_length];
    783 		b32 map_channels = (local_flags & BeamformerShaderFilterFlags_MapChannels) != 0;
    784 
    785 		u32 index = shader == BeamformerShaderKind_Filter ? BeamformerComputeUBOKind_Filter
    786 		                                                  : BeamformerComputeUBOKind_Demodulate;
    787 		glBindBufferBase(GL_UNIFORM_BUFFER,        0, cp->ubos[index]);
    788 		glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[output_ssbo_idx]);
    789 
    790 		if (!map_channels)
    791 			glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]);
    792 
    793 		GLenum kind = cp->filters[sp->filter_slot].parameters.complex? GL_RG32F : GL_R32F;
    794 		glBindImageTexture(0, cp->filters[sp->filter_slot].texture, 0, 0, 0, GL_READ_ONLY, kind);
    795 
    796 		if (map_channels)
    797 			glBindImageTexture(1, cp->textures[BeamformerComputeTextureKind_ChannelMapping], 0, 0, 0, GL_READ_ONLY, GL_R16I);
    798 
    799 		glDispatchCompute(cp->demod_dispatch.x, cp->demod_dispatch.y, cp->demod_dispatch.z);
    800 		glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);
    801 
    802 		cc->last_output_ssbo_index = !cc->last_output_ssbo_index;
    803 	}break;
    804 	case BeamformerShaderKind_MinMax:{
    805 		for (i32 i = 1; i < frame->mips; i++) {
    806 			glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY,  GL_RG32F);
    807 			glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F);
    808 			glProgramUniform1i(cc->programs[shader], MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i);
    809 
    810 			u32 width  = (u32)frame->dim.x >> i;
    811 			u32 height = (u32)frame->dim.y >> i;
    812 			u32 depth  = (u32)frame->dim.z >> i;
    813 			glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32));
    814 			glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
    815 		}
    816 	}break;
    817 	case BeamformerShaderKind_DAS:{
    818 		BeamformerDASUBO *ubo = &cp->das_ubo_data;
    819 
    820 		i32 local_flags = match_vector[shader_descriptor->match_vector_length];
    821 		b32 fast        = (local_flags & BeamformerShaderDASFlags_Fast)   != 0;
    822 		b32 sparse      = (local_flags & BeamformerShaderDASFlags_Sparse) != 0;
    823 
    824 		if (fast) {
    825 			glClearTexImage(frame->texture, 0, GL_RED, GL_FLOAT, 0);
    826 			glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT);
    827 			glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_READ_WRITE, cp->iq_pipeline ? GL_RG32F : GL_R32F);
    828 		} else {
    829 			glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_WRITE_ONLY, cp->iq_pipeline ? GL_RG32F : GL_R32F);
    830 		}
    831 
    832 		u32 sparse_texture = cp->textures[BeamformerComputeTextureKind_SparseElements];
    833 		if (!sparse) sparse_texture = 0;
    834 
    835 		glBindBufferBase(GL_UNIFORM_BUFFER, 0, cp->ubos[BeamformerComputeUBOKind_DAS]);
    836 		glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx], 0, cp->rf_size);
    837 		glBindImageTexture(1, sparse_texture, 0, 0, 0, GL_READ_ONLY, GL_R16I);
    838 		glBindImageTexture(2, cp->textures[BeamformerComputeTextureKind_FocalVectors], 0, 0, 0, GL_READ_ONLY, GL_RG32F);
    839 
    840 		glProgramUniform1ui(program, DAS_CYCLE_T_UNIFORM_LOC, cycle_t++);
    841 
    842 		if (fast) {
    843 			i32 loop_end;
    844 			if (ubo->shader_kind == BeamformerDASKind_RCA_VLS ||
    845 			    ubo->shader_kind == BeamformerDASKind_RCA_TPW)
    846 			{
    847 				/* NOTE(rnp): to avoid repeatedly sampling the whole focal vectors
    848 				 * texture we loop over transmits for VLS/TPW */
    849 				loop_end = (i32)ubo->acquisition_count;
    850 			} else {
    851 				loop_end = (i32)ubo->channel_count;
    852 			}
    853 			f32 percent_per_step = 1.0f / (f32)loop_end;
    854 			cc->processing_progress = -percent_per_step;
    855 			for (i32 index = 0; index < loop_end; index++) {
    856 				cc->processing_progress += percent_per_step;
    857 				/* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */
    858 				glFinish();
    859 				glProgramUniform1i(program, DAS_FAST_CHANNEL_UNIFORM_LOC, index);
    860 				glDispatchCompute((u32)ceil_f32((f32)frame->dim.x / DAS_LOCAL_SIZE_X),
    861 				                  (u32)ceil_f32((f32)frame->dim.y / DAS_LOCAL_SIZE_Y),
    862 				                  (u32)ceil_f32((f32)frame->dim.z / DAS_LOCAL_SIZE_Z));
    863 				glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
    864 			}
    865 		} else {
    866 			#if 1
    867 			/* TODO(rnp): compute max_points_per_dispatch based on something like a
    868 			 * transmit_count * channel_count product */
    869 			u32 max_points_per_dispatch = KB(64);
    870 			struct compute_cursor cursor = start_compute_cursor(frame->dim, max_points_per_dispatch);
    871 			f32 percent_per_step = (f32)cursor.points_per_dispatch / (f32)cursor.total_points;
    872 			cc->processing_progress = -percent_per_step;
    873 			for (iv3 offset = {0};
    874 			     !compute_cursor_finished(&cursor);
    875 			     offset = step_compute_cursor(&cursor))
    876 			{
    877 				cc->processing_progress += percent_per_step;
    878 				/* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */
    879 				glFinish();
    880 				glProgramUniform3iv(program, DAS_VOXEL_OFFSET_UNIFORM_LOC, 1, offset.E);
    881 				glDispatchCompute(cursor.dispatch.x, cursor.dispatch.y, cursor.dispatch.z);
    882 			}
    883 			#else
    884 			/* NOTE(rnp): use this for testing tiling code. The performance of the above path
    885 			 * should be the same as this path if everything is working correctly */
    886 			iv3 compute_dim_offset = {0};
    887 			glProgramUniform3iv(program, DAS_VOXEL_OFFSET_UNIFORM_LOC, 1, compute_dim_offset.E);
    888 			glDispatchCompute((u32)ceil_f32((f32)dim.x / DAS_LOCAL_SIZE_X),
    889 			                  (u32)ceil_f32((f32)dim.y / DAS_LOCAL_SIZE_Y),
    890 			                  (u32)ceil_f32((f32)dim.z / DAS_LOCAL_SIZE_Z));
    891 			#endif
    892 		}
    893 		glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT|GL_SHADER_IMAGE_ACCESS_BARRIER_BIT);
    894 	}break;
    895 	case BeamformerShaderKind_Sum:{
    896 		u32 aframe_index = ctx->averaged_frame_index % ARRAY_COUNT(ctx->averaged_frames);
    897 		BeamformerFrame *aframe = ctx->averaged_frames + aframe_index;
    898 		aframe->id              = ctx->averaged_frame_index;
    899 		atomic_store_u32(&aframe->ready_to_present, 0);
    900 		/* TODO(rnp): hack we need a better way of specifying which frames to sum;
    901 		 * this is fine for rolling averaging but what if we want to do something else */
    902 		assert(frame >= ctx->beamform_frames);
    903 		assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames));
    904 		u32 base_index   = (u32)(frame - ctx->beamform_frames);
    905 		u32 to_average   = (u32)cp->average_frames;
    906 		u32 frame_count  = 0;
    907 		u32 *in_textures = push_array(&arena, u32, BeamformerMaxSavedFrames);
    908 		ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average);
    909 		for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi))
    910 			in_textures[frame_count++] = it->texture;
    911 
    912 		assert(to_average == frame_count);
    913 
    914 		do_sum_shader(cc, in_textures, frame_count, 1 / (f32)frame_count, aframe->texture, aframe->dim);
    915 		aframe->min_coordinate  = frame->min_coordinate;
    916 		aframe->max_coordinate  = frame->max_coordinate;
    917 		aframe->compound_count  = frame->compound_count;
    918 		aframe->das_kind        = frame->das_kind;
    919 	}break;
    920 	InvalidDefaultCase;
    921 	}
    922 }
    923 
    924 function void
    925 stream_push_shader_header(Stream *s, ShaderReloadContext *ctx)
    926 {
    927 	BeamformerReloadableShaderInfo *rsi = beamformer_reloadable_shader_infos + ctx->reloadable_info_index;
    928 
    929 	stream_append_s8s(s, s8("#version 460 core\n\n"), ctx->header);
    930 
    931 	switch (rsi->kind) {
    932 	case BeamformerShaderKind_Filter:{
    933 		stream_append_s8(s, s8(""
    934 		"layout(local_size_x = " str(FILTER_LOCAL_SIZE_X) ", "
    935 		       "local_size_y = " str(FILTER_LOCAL_SIZE_Y) ", "
    936 		       "local_size_z = " str(FILTER_LOCAL_SIZE_Z) ") in;\n\n"
    937 		));
    938 	}break;
    939 	case BeamformerShaderKind_DAS:{
    940 		stream_append_s8(s, s8(""
    941 		"layout(local_size_x = " str(DAS_LOCAL_SIZE_X) ", "
    942 		       "local_size_y = " str(DAS_LOCAL_SIZE_Y) ", "
    943 		       "local_size_z = " str(DAS_LOCAL_SIZE_Z) ") in;\n\n"
    944 		"layout(location = " str(DAS_VOXEL_OFFSET_UNIFORM_LOC) ") uniform ivec3 u_voxel_offset;\n"
    945 		"layout(location = " str(DAS_CYCLE_T_UNIFORM_LOC)      ") uniform uint  u_cycle_t;\n"
    946 		"layout(location = " str(DAS_FAST_CHANNEL_UNIFORM_LOC) ") uniform int   u_channel;\n\n"
    947 		));
    948 
    949 		#define X(k, id, ...) "#define ShaderKind_" #k " " #id "\n"
    950 		stream_append_s8s(s, s8(DAS_SHADER_KIND_LIST), s8("\n"));
    951 		#undef X
    952 	}break;
    953 	case BeamformerShaderKind_Decode:{
    954 		stream_append_s8s(s, s8(""
    955 		"layout(local_size_x = " str(DECODE_LOCAL_SIZE_X) ", "
    956 		       "local_size_y = " str(DECODE_LOCAL_SIZE_Y) ", "
    957 		       "local_size_z = " str(DECODE_LOCAL_SIZE_Z) ") in;\n\n"
    958 		"layout(location = " str(DECODE_FIRST_PASS_UNIFORM_LOC) ") uniform bool u_first_pass;\n\n"
    959 		));
    960 	}break;
    961 	case BeamformerShaderKind_MinMax:{
    962 		stream_append_s8(s, s8("layout(location = " str(MIN_MAX_MIPS_LEVEL_UNIFORM_LOC)
    963 		                       ") uniform int u_mip_map;\n\n"));
    964 	}break;
    965 	case BeamformerShaderKind_Sum:{
    966 		stream_append_s8(s, s8("layout(location = " str(SUM_PRESCALE_UNIFORM_LOC)
    967 		                       ") uniform float u_sum_prescale = 1.0;\n\n"));
    968 	}break;
    969 	default:{}break;
    970 	}
    971 }
    972 
    973 function s8
    974 shader_text_with_header(ShaderReloadContext *ctx, s8 filepath, Arena *arena)
    975 {
    976 	Stream sb = arena_stream(*arena);
    977 	stream_push_shader_header(&sb, ctx);
    978 	stream_append_s8(&sb, s8("\n#line 1\n"));
    979 
    980 	s8 result = arena_stream_commit(arena, &sb);
    981 	if (filepath.len > 0) {
    982 		s8 file = os_read_whole_file(arena, (c8 *)filepath.data);
    983 		assert(file.data == result.data + result.len);
    984 		result.len += file.len;
    985 	}
    986 
    987 	return result;
    988 }
    989 
    990 /* NOTE(rnp): currently this function is only handling rendering shaders.
    991  * look at reload_compute_shader for compute shaders */
    992 DEBUG_EXPORT BEAMFORMER_RELOAD_SHADER_FN(beamformer_reload_shader)
    993 {
    994 	BeamformerCtx *ctx = src->beamformer_context;
    995 
    996 	i32 shader_count = 1;
    997 	ShaderReloadContext *link = src->link;
    998 	while (link != src) { shader_count++; link = link->link; }
    999 
   1000 	s8  *shader_texts = push_array(&arena, s8,  shader_count);
   1001 	u32 *shader_types = push_array(&arena, u32, shader_count);
   1002 
   1003 	i32 index = 0;
   1004 	do {
   1005 		s8 filepath = {0};
   1006 		if (link->reloadable_info_index >= 0) filepath = path;
   1007 		shader_texts[index] = shader_text_with_header(link, filepath, &arena);
   1008 		shader_types[index] = link->gl_type;
   1009 		index++;
   1010 		link = link->link;
   1011 	} while (link != src);
   1012 
   1013 	BeamformerReloadableShaderInfo *rsi = beamformer_reloadable_shader_infos + src->reloadable_info_index;
   1014 	assert(rsi->kind == BeamformerShaderKind_Render3D);
   1015 
   1016 	u32 *shader = &ctx->frame_view_render_context.shader;
   1017 	glDeleteProgram(*shader);
   1018 	*shader = load_shader(&ctx->os, arena, shader_texts, shader_types, shader_count, shader_name);
   1019 	ctx->frame_view_render_context.updated = 1;
   1020 
   1021 	return 1;
   1022 }
   1023 
   1024 function void
   1025 reload_compute_shader(BeamformerCtx *ctx, ShaderReloadContext *src, Arena arena)
   1026 {
   1027 	BeamformerComputeContext       *cc  = &ctx->compute_context;
   1028 	BeamformerReloadableShaderInfo *rsi = beamformer_reloadable_shader_infos + src->reloadable_info_index;
   1029 	BeamformerShaderDescriptor     *sd  = beamformer_shader_descriptors + rsi->kind;
   1030 
   1031 	Stream status = stream_alloc(&arena, 128);
   1032 	u32 completed     = 0;
   1033 	u32 total_shaders = (u32)(sd->one_past_last_match_vector_index - sd->first_match_vector_index);
   1034 	for (i32 i = 0; i < rsi->sub_shader_descriptor_index_count; i++) {
   1035 		BeamformerShaderDescriptor *ssd  = beamformer_shader_descriptors + rsi->sub_shader_descriptor_indices[i];
   1036 		total_shaders += (u32)(ssd->one_past_last_match_vector_index - ssd->first_match_vector_index);
   1037 	}
   1038 
   1039 	s8 path = push_s8_from_parts(&arena, ctx->os.path_separator, s8("shaders"),
   1040 	                             beamformer_reloadable_shader_files[src->reloadable_info_index]);
   1041 	s8 file_text  = os_read_whole_file(&arena, (c8 *)path.data);
   1042 	Stream shader = arena_stream(arena);
   1043 
   1044 	stream_push_shader_header(&shader, src);
   1045 
   1046 	stream_append_s8(&shader, beamformer_shader_local_header_strings[src->reloadable_info_index]);
   1047 
   1048 	i32 save_point = shader.widx;
   1049 	for (i32 sub_index = -1; sub_index < rsi->sub_shader_descriptor_index_count; sub_index++) {
   1050 		shader.widx = save_point;
   1051 
   1052 		if (sub_index != -1)
   1053 			sd = beamformer_shader_descriptors + rsi->sub_shader_descriptor_indices[sub_index];
   1054 
   1055 		i32 *hvector = beamformer_shader_header_vectors[sd - beamformer_shader_descriptors];
   1056 		for (i32 index = 0; index < sd->header_vector_length; index++)
   1057 			stream_append_s8s(&shader, beamformer_shader_global_header_strings[hvector[index]], s8("\n"));
   1058 
   1059 		i32 instance_save_point = shader.widx;
   1060 		arena_commit(&arena, instance_save_point);
   1061 		TempArena arena_save = begin_temp_arena(&arena);
   1062 
   1063 		for (i32 instance = sd->first_match_vector_index;
   1064 		     instance < sd->one_past_last_match_vector_index;
   1065 		     instance++)
   1066 		{
   1067 			shader.widx = instance_save_point;
   1068 			end_temp_arena(arena_save);
   1069 
   1070 			i32 *match_vector = beamformer_shader_match_vectors[instance];
   1071 			for (i32 index = 0; index < sd->match_vector_length; index++) {
   1072 				stream_append_s8s(&shader, s8("#define "), beamformer_shader_descriptor_header_strings[hvector[index]], s8(" ("));
   1073 				stream_append_i64(&shader, match_vector[index]);
   1074 				stream_append_s8(&shader, s8(")\n"));
   1075 			}
   1076 
   1077 			if (sd->has_local_flags) {
   1078 				stream_append_s8(&shader, s8("#define ShaderFlags (0x"));
   1079 				stream_append_hex_u64(&shader, (u64)match_vector[sd->match_vector_length]);
   1080 				stream_append_s8(&shader, s8(")\n"));
   1081 			}
   1082 
   1083 			stream_append_s8s(&shader, s8("\n#line 1\n"), file_text);
   1084 
   1085 			arena_commit(&arena, shader.widx - instance_save_point);
   1086 
   1087 			s8 shader_text = stream_to_s8(&shader);
   1088 			/* TODO(rnp): instance name */
   1089 			s8 shader_name = beamformer_shader_names[rsi->kind];
   1090 			glDeleteProgram(cc->programs[instance]);
   1091 			cc->programs[instance] = load_shader(&ctx->os, arena, &shader_text, &src->gl_type, 1, shader_name);
   1092 
   1093 			status.widx = 0;
   1094 			stream_append_s8s(&status, s8("\r\x1b[2Kloaded shader "), shader_name, s8(": ["));
   1095 			stream_append_u64(&status, ++completed);
   1096 			stream_append_s8s(&status, s8("/"));
   1097 			stream_append_u64(&status, total_shaders);
   1098 			stream_append_s8s(&status, s8("]"));
   1099 			os_write_file(ctx->os.error_handle, stream_to_s8(&status));
   1100 		}
   1101 	}
   1102 	os_write_file(ctx->os.error_handle, s8("\n"));
   1103 }
   1104 
   1105 function void
   1106 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena, iptr gl_context)
   1107 {
   1108 	BeamformerComputeContext *cs = &ctx->compute_context;
   1109 	BeamformerSharedMemory   *sm = ctx->shared_memory.region;
   1110 
   1111 	BeamformWork *work = beamform_work_queue_pop(q);
   1112 	while (work) {
   1113 		b32 can_commit = 1;
   1114 		switch (work->kind) {
   1115 		case BeamformerWorkKind_ReloadShader:{
   1116 			reload_compute_shader(ctx, work->shader_reload_context, *arena);
   1117 			if (ctx->latest_frame && !sm->live_imaging_parameters.active) {
   1118 				fill_frame_compute_work(ctx, work, ctx->latest_frame->view_plane_tag, 0, 0);
   1119 				can_commit = 0;
   1120 			}
   1121 		}break;
   1122 		case BeamformerWorkKind_ExportBuffer:{
   1123 			/* TODO(rnp): better way of handling DispatchCompute barrier */
   1124 			post_sync_barrier(&ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute, sm->locks);
   1125 			os_shared_memory_region_lock(&ctx->shared_memory, sm->locks, (i32)work->lock, (u32)-1);
   1126 			BeamformerExportContext *ec = &work->export_context;
   1127 			switch (ec->kind) {
   1128 			case BeamformerExportKind_BeamformedData:{
   1129 				BeamformerFrame *frame = ctx->latest_frame;
   1130 				if (frame) {
   1131 					assert(frame->ready_to_present);
   1132 					u32 texture  = frame->texture;
   1133 					iv3 dim      = frame->dim;
   1134 					u32 out_size = (u32)dim.x * (u32)dim.y * (u32)dim.z * 2 * sizeof(f32);
   1135 					if (out_size <= ec->size) {
   1136 						glGetTextureImage(texture, 0, GL_RG, GL_FLOAT, (i32)out_size,
   1137 						                  beamformer_shared_memory_scratch_arena(sm).beg);
   1138 					}
   1139 				}
   1140 			}break;
   1141 			case BeamformerExportKind_Stats:{
   1142 				ComputeTimingTable *table = ctx->compute_timing_table;
   1143 				/* NOTE(rnp): do a little spin to let this finish updating */
   1144 				while (table->write_index != atomic_load_u32(&table->read_index));
   1145 				ComputeShaderStats *stats = ctx->compute_shader_stats;
   1146 				if (sizeof(stats->table) <= ec->size)
   1147 					mem_copy(beamformer_shared_memory_scratch_arena(sm).beg, &stats->table, sizeof(stats->table));
   1148 			}break;
   1149 			InvalidDefaultCase;
   1150 			}
   1151 			os_shared_memory_region_unlock(&ctx->shared_memory, sm->locks, (i32)work->lock);
   1152 			post_sync_barrier(&ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync, sm->locks);
   1153 		}break;
   1154 		case BeamformerWorkKind_CreateFilter:{
   1155 			/* TODO(rnp): this should probably get deleted and moved to lazy loading */
   1156 			BeamformerCreateFilterContext *fctx = &work->create_filter_context;
   1157 			u32 block = fctx->parameter_block;
   1158 			u32 slot  = fctx->filter_slot;
   1159 			BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena);
   1160 			beamformer_filter_update(cp->filters + slot, fctx->kind, fctx->parameters, block, slot, *arena);
   1161 		}break;
   1162 		case BeamformerWorkKind_ComputeIndirect:{
   1163 			fill_frame_compute_work(ctx, work, work->compute_indirect_context.view_plane,
   1164 			                        work->compute_indirect_context.parameter_block, 1);
   1165 		} /* FALLTHROUGH */
   1166 		case BeamformerWorkKind_Compute:{
   1167 			DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[0], GL_RG32F, GL_RG, GL_FLOAT, 0);)
   1168 			DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[1], GL_RG32F, GL_RG, GL_FLOAT, 0);)
   1169 			DEBUG_DECL(glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);)
   1170 
   1171 			push_compute_timing_info(ctx->compute_timing_table,
   1172 			                         (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin});
   1173 
   1174 			BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_block, arena);
   1175 			if (beamformer_parameter_block_dirty(sm, work->compute_context.parameter_block)) {
   1176 				u32 block = work->compute_context.parameter_block;
   1177 				beamformer_commit_parameter_block(ctx, cp, block, *arena);
   1178 				atomic_store_u32(&ctx->ui_dirty_parameter_blocks, (u32)(ctx->beamform_work_queue != q) << block);
   1179 			}
   1180 
   1181 			post_sync_barrier(&ctx->shared_memory, work->lock, sm->locks);
   1182 
   1183 			atomic_store_u32(&cs->processing_compute, 1);
   1184 			start_renderdoc_capture(gl_context);
   1185 
   1186 			BeamformerFrame *frame = work->compute_context.frame;
   1187 
   1188 			GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F;
   1189 			if (!beamformer_frame_compatible(frame, cp->output_points, gl_kind))
   1190 				alloc_beamform_frame(&ctx->gl, frame, cp->output_points, gl_kind, s8("Beamformed_Data"), *arena);
   1191 
   1192 			frame->min_coordinate  = cp->min_coordinate;
   1193 			frame->max_coordinate  = cp->max_coordinate;
   1194 			frame->das_kind        = cp->das_ubo_data.shader_kind;
   1195 			frame->compound_count  = cp->das_ubo_data.acquisition_count;
   1196 
   1197 			BeamformerComputeContext  *cc       = &ctx->compute_context;
   1198 			BeamformerComputePipeline *pipeline = &cp->pipeline;
   1199 			/* NOTE(rnp): first stage requires access to raw data buffer directly so we break
   1200 			 * it out into a separate step. This way data can get released as soon as possible */
   1201 			if (pipeline->shader_count > 0) {
   1202 				BeamformerRFBuffer *rf = &cs->rf_buffer;
   1203 				u32 slot = rf->compute_index % countof(rf->compute_syncs);
   1204 
   1205 				/* NOTE(rnp): compute indirect is used when uploading data. in this case the thread
   1206 				 * must wait on an upload fence. if the fence doesn't yet exist the thread must wait */
   1207 				if (work->kind == BeamformerWorkKind_ComputeIndirect)
   1208 					spin_wait(!atomic_load_u64(rf->upload_syncs + slot));
   1209 
   1210 				if (rf->upload_syncs[slot]) {
   1211 					rf->compute_index++;
   1212 					glWaitSync(rf->upload_syncs[slot], 0, GL_TIMEOUT_IGNORED);
   1213 					glDeleteSync(rf->upload_syncs[slot]);
   1214 				} else {
   1215 					slot = (rf->compute_index - 1) % countof(rf->compute_syncs);
   1216 				}
   1217 
   1218 				glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, rf->ssbo, slot * rf->active_rf_size, rf->active_rf_size);
   1219 
   1220 				glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[0]);
   1221 				do_compute_shader(ctx, cp, frame, pipeline->shaders[0], pipeline->program_indices[0],
   1222 				                  pipeline->parameters + 0, *arena);
   1223 				glEndQuery(GL_TIME_ELAPSED);
   1224 
   1225 				if (work->kind == BeamformerWorkKind_ComputeIndirect) {
   1226 					rf->compute_syncs[slot] = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
   1227 					rf->upload_syncs[slot]  = 0;
   1228 					memory_write_barrier();
   1229 				}
   1230 			}
   1231 
   1232 			b32 did_sum_shader = 0;
   1233 			for (u32 i = 1; i < pipeline->shader_count; i++) {
   1234 				did_sum_shader |= pipeline->shaders[i] == BeamformerShaderKind_Sum;
   1235 				glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[i]);
   1236 				do_compute_shader(ctx, cp, frame, pipeline->shaders[i], pipeline->program_indices[i],
   1237 				                  pipeline->parameters + i, *arena);
   1238 				glEndQuery(GL_TIME_ELAPSED);
   1239 			}
   1240 
   1241 			/* NOTE(rnp): the first of these blocks until work completes */
   1242 			for (u32 i = 0; i < pipeline->shader_count; i++) {
   1243 				ComputeTimingInfo info = {0};
   1244 				info.kind   = ComputeTimingInfoKind_Shader;
   1245 				info.shader = pipeline->shaders[i];
   1246 				glGetQueryObjectui64v(cc->shader_timer_ids[i], GL_QUERY_RESULT, &info.timer_count);
   1247 				push_compute_timing_info(ctx->compute_timing_table, info);
   1248 			}
   1249 			cs->processing_progress = 1;
   1250 
   1251 			frame->ready_to_present = 1;
   1252 			if (did_sum_shader) {
   1253 				u32 aframe_index = (ctx->averaged_frame_index % countof(ctx->averaged_frames));
   1254 				ctx->averaged_frames[aframe_index].view_plane_tag  = frame->view_plane_tag;
   1255 				ctx->averaged_frames[aframe_index].ready_to_present = 1;
   1256 				atomic_add_u32(&ctx->averaged_frame_index, 1);
   1257 				atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index));
   1258 			} else {
   1259 				atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame);
   1260 			}
   1261 			cs->processing_compute  = 0;
   1262 
   1263 			push_compute_timing_info(ctx->compute_timing_table,
   1264 			                         (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd});
   1265 
   1266 			end_renderdoc_capture(gl_context);
   1267 		}break;
   1268 		InvalidDefaultCase;
   1269 		}
   1270 
   1271 		if (can_commit) {
   1272 			beamform_work_queue_pop_commit(q);
   1273 			work = beamform_work_queue_pop(q);
   1274 		}
   1275 	}
   1276 }
   1277 
   1278 function void
   1279 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats)
   1280 {
   1281 	/* TODO(rnp): we do not currently do anything to handle the potential for a half written
   1282 	 * info item. this could result in garbage entries but they shouldn't really matter */
   1283 
   1284 	u32 target = atomic_load_u32(&t->write_index);
   1285 	u32 stats_index = (stats->latest_frame_index + 1) % countof(stats->table.times);
   1286 
   1287 	static_assert(BeamformerShaderKind_Count + 1 <= 32, "timing coalescence bitfield test");
   1288 	u32 seen_info_test = 0;
   1289 
   1290 	while (t->read_index != target) {
   1291 		ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)];
   1292 		switch (info.kind) {
   1293 		case ComputeTimingInfoKind_ComputeFrameBegin:{
   1294 			assert(t->compute_frame_active == 0);
   1295 			t->compute_frame_active = 1;
   1296 			/* NOTE(rnp): allow multiple instances of same shader to accumulate */
   1297 			mem_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index]));
   1298 		}break;
   1299 		case ComputeTimingInfoKind_ComputeFrameEnd:{
   1300 			assert(t->compute_frame_active == 1);
   1301 			t->compute_frame_active = 0;
   1302 			stats->latest_frame_index = stats_index;
   1303 			stats_index = (stats_index + 1) % countof(stats->table.times);
   1304 		}break;
   1305 		case ComputeTimingInfoKind_Shader:{
   1306 			stats->table.times[stats_index][info.shader] += (f32)info.timer_count / 1.0e9f;
   1307 			seen_info_test |= (1u << info.shader);
   1308 		}break;
   1309 		case ComputeTimingInfoKind_RF_Data:{
   1310 			stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas);
   1311 			f32 delta = (f32)(info.timer_count - stats->last_rf_timer_count) / 1.0e9f;
   1312 			stats->table.rf_time_deltas[stats->latest_rf_index] = delta;
   1313 			stats->last_rf_timer_count = info.timer_count;
   1314 			seen_info_test |= (1 << BeamformerShaderKind_Count);
   1315 		}break;
   1316 		}
   1317 		/* NOTE(rnp): do this at the end so that stats table is always in a consistent state */
   1318 		atomic_add_u32(&t->read_index, 1);
   1319 	}
   1320 
   1321 	if (seen_info_test) {
   1322 		for EachEnumValue(BeamformerShaderKind, shader) {
   1323 			if (seen_info_test & (1 << shader)) {
   1324 				f32 sum = 0;
   1325 				for EachElement(stats->table.times, i)
   1326 					sum += stats->table.times[i][shader];
   1327 				stats->average_times[shader] = sum / countof(stats->table.times);
   1328 			}
   1329 		}
   1330 
   1331 		if (seen_info_test & (1 << BeamformerShaderKind_Count)) {
   1332 			f32 sum = 0;
   1333 			for EachElement(stats->table.rf_time_deltas, i)
   1334 				sum += stats->table.rf_time_deltas[i];
   1335 			stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas);
   1336 		}
   1337 	}
   1338 }
   1339 
   1340 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute)
   1341 {
   1342 	BeamformerCtx *ctx         = (BeamformerCtx *)user_context;
   1343 	BeamformerSharedMemory *sm = ctx->shared_memory.region;
   1344 	complete_queue(ctx, &sm->external_work_queue, arena, gl_context);
   1345 	complete_queue(ctx, ctx->beamform_work_queue, arena, gl_context);
   1346 }
   1347 
   1348 function void
   1349 beamformer_rf_buffer_allocate(BeamformerRFBuffer *rf, u32 rf_size, Arena arena)
   1350 {
   1351 	assert((rf_size % 64) == 0);
   1352 	glUnmapNamedBuffer(rf->ssbo);
   1353 	glDeleteBuffers(1, &rf->ssbo);
   1354 	glCreateBuffers(1, &rf->ssbo);
   1355 
   1356 	glNamedBufferStorage(rf->ssbo, countof(rf->compute_syncs) * rf_size, 0,
   1357 	                     GL_DYNAMIC_STORAGE_BIT|GL_MAP_WRITE_BIT);
   1358 	LABEL_GL_OBJECT(GL_BUFFER, rf->ssbo, s8("Raw_RF_SSBO"));
   1359 	rf->size = rf_size;
   1360 }
   1361 
   1362 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
   1363 {
   1364 	BeamformerSharedMemory *sm = ctx->shared_memory->region;
   1365 
   1366 	BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace;
   1367 	BeamformerSharedMemoryLockKind upload_lock  = BeamformerSharedMemoryLockKind_UploadRF;
   1368 	if (sm->locks[upload_lock] &&
   1369 	    os_shared_memory_region_lock(ctx->shared_memory, sm->locks, (i32)scratch_lock, (u32)-1))
   1370 	{
   1371 		BeamformerRFBuffer *rf = ctx->rf_buffer;
   1372 		rf->active_rf_size = (u32)round_up_to(sm->scratch_rf_size, 64);
   1373 		if (rf->size < rf->active_rf_size)
   1374 			beamformer_rf_buffer_allocate(rf, rf->active_rf_size, arena);
   1375 
   1376 		u32 slot = rf->insertion_index++ % countof(rf->compute_syncs);
   1377 
   1378 		/* NOTE(rnp): if the rest of the code is functioning then the first
   1379 		 * time the compute thread processes an upload it must have gone
   1380 		 * through this path. therefore it is safe to spin until it gets processed */
   1381 		spin_wait(atomic_load_u64(rf->upload_syncs + slot));
   1382 
   1383 		if (rf->compute_syncs[slot]) {
   1384 			GLenum sync_result = glClientWaitSync(rf->compute_syncs[slot], 0, 1000000000);
   1385 			if (sync_result == GL_TIMEOUT_EXPIRED || sync_result == GL_WAIT_FAILED) {
   1386 				// TODO(rnp): what do?
   1387 			}
   1388 			glDeleteSync(rf->compute_syncs[slot]);
   1389 		}
   1390 
   1391 		/* NOTE(rnp): nVidia's drivers really don't play nice with persistant mapping,
   1392 		 * at least when it is a big as this one wants to be. mapping and unmapping the
   1393 		 * desired range each time doesn't seem to introduce any performance hit */
   1394 		u32 access = GL_MAP_WRITE_BIT|GL_MAP_FLUSH_EXPLICIT_BIT|GL_MAP_UNSYNCHRONIZED_BIT;
   1395 		u8 *buffer = glMapNamedBufferRange(rf->ssbo, slot * rf->active_rf_size, (i32)rf->active_rf_size, access);
   1396 
   1397 		mem_copy(buffer, beamformer_shared_memory_scratch_arena(sm).beg, rf->active_rf_size);
   1398 		os_shared_memory_region_unlock(ctx->shared_memory, sm->locks, (i32)scratch_lock);
   1399 		post_sync_barrier(ctx->shared_memory, upload_lock, sm->locks);
   1400 
   1401 		glFlushMappedNamedBufferRange(rf->ssbo, 0, (i32)rf->active_rf_size);
   1402 		glUnmapNamedBuffer(rf->ssbo);
   1403 
   1404 		rf->upload_syncs[slot]  = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
   1405 		rf->compute_syncs[slot] = 0;
   1406 		memory_write_barrier();
   1407 
   1408 		os_wake_waiters(ctx->compute_worker_sync);
   1409 
   1410 		ComputeTimingInfo info = {.kind = ComputeTimingInfoKind_RF_Data};
   1411 		glGetQueryObjectui64v(rf->data_timestamp_query, GL_QUERY_RESULT, &info.timer_count);
   1412 		glQueryCounter(rf->data_timestamp_query, GL_TIMESTAMP);
   1413 		push_compute_timing_info(ctx->compute_timing_table, info);
   1414 	}
   1415 }
   1416 
   1417 #include "ui.c"
   1418 
   1419 DEBUG_EXPORT BEAMFORMER_FRAME_STEP_FN(beamformer_frame_step)
   1420 {
   1421 	dt_for_frame = input->dt;
   1422 
   1423 	if (IsWindowResized()) {
   1424 		ctx->window_size.h = GetScreenHeight();
   1425 		ctx->window_size.w = GetScreenWidth();
   1426 	}
   1427 
   1428 	coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats);
   1429 
   1430 	if (input->executable_reloaded) {
   1431 		ui_init(ctx, ctx->ui_backing_store);
   1432 		DEBUG_DECL(start_frame_capture = ctx->os.start_frame_capture);
   1433 		DEBUG_DECL(end_frame_capture   = ctx->os.end_frame_capture);
   1434 	}
   1435 
   1436 	BeamformerSharedMemory *sm = ctx->shared_memory.region;
   1437 	if (sm->locks[BeamformerSharedMemoryLockKind_UploadRF] != 0)
   1438 		os_wake_waiters(&ctx->os.upload_worker.sync_variable);
   1439 
   1440 	BeamformerFrame        *frame = ctx->latest_frame;
   1441 	BeamformerViewPlaneTag  tag   = frame? frame->view_plane_tag : 0;
   1442 	draw_ui(ctx, input, frame, tag);
   1443 
   1444 	ctx->frame_view_render_context.updated = 0;
   1445 
   1446 	if (WindowShouldClose())
   1447 		ctx->should_exit = 1;
   1448 }
   1449 
   1450 /* NOTE(rnp): functions defined in these shouldn't be visible to the whole program */
   1451 #if _DEBUG
   1452   #if OS_LINUX
   1453     #include "os_linux.c"
   1454   #elif OS_WINDOWS
   1455     #include "os_win32.c"
   1456   #endif
   1457 #endif