ogl_beamforming

Ultrasound Beamforming Implemented with OpenGL
git clone anongit@rnpnr.xyz:ogl_beamforming.git
Log | Files | Refs | Feed | Submodules | README | LICENSE

ogl_beamformer_lib.c (27884B)


      1 /* See LICENSE for license details. */
      2 #define BEAMFORMER_IMPORT static
      3 
      4 #ifndef BASE_PLATFORM_H
      5 #define BASE_PLATFORM_NO_MAIN 1
      6 #define BASE_EXPORT static
      7 #endif
      8 
      9 #include "../util.h"
     10 
     11 #include "../generated/beamformer.c"
     12 #include "ogl_beamformer_lib_base.h"
     13 
     14 #if OS_LINUX
     15 #include "../base_linux.c"
     16 #elif OS_WINDOWS
     17 #include "../base_win32.c"
     18 
     19 W32(iptr) OpenFileMappingA(u32, b32, c8 *);
     20 
     21 #else
     22 #error Unsupported Platform
     23 #endif
     24 
     25 #include "../beamformer_compute_stats.c"
     26 #include "../beamformer_shared_memory.c"
     27 
     28 global struct {
     29 	BeamformerSharedMemory *bp;
     30 	b32                     new_rf;
     31 	i32                     timeout_ms;
     32 	BeamformerLibErrorKind  last_error;
     33 } g_beamformer_library_context;
     34 
     35 #if OS_LINUX
     36 
     37 function str8
     38 os_open_shared_memory_area(char *name)
     39 {
     40 	str8 result = {0};
     41 	i32 fd = shm_open(name, O_RDWR, S_IRUSR|S_IWUSR);
     42 	if (fd > 0) {
     43 		struct stat sb;
     44 		if (fstat(fd, &sb) != -1) {
     45 			void *new = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
     46 			if (new != MAP_FAILED) {
     47 				result.data   = new;
     48 				result.length = sb.st_size;
     49 			}
     50 		}
     51 		close(fd);
     52 	}
     53 	return result;
     54 }
     55 
     56 function void
     57 os_close_shared_memory_area(void *memory, i64 size)
     58 {
     59 	munmap(memory, size);
     60 }
     61 
     62 #elif OS_WINDOWS
     63 
     64 W32(u64) VirtualQuery(void *base_address, void *memory_basic_info, u64 memory_basic_info_size);
     65 W32(b32) UnmapViewOfFile(void *);
     66 
     67 function b32
     68 os_reserve_region_locks(void)
     69 {
     70 	u8 buffer[1024];
     71 	Stream sb = {.data = buffer, .cap = countof(buffer)};
     72 	stream_append_str8(&sb, str8(OS_SHARED_MEMORY_NAME "_lock_"));
     73 
     74 	i32 start_index    = sb.widx;
     75 	u32 reserved_count = 0;
     76 	for EachElement(os_w32_shared_memory_semaphores, it) {
     77 		stream_reset(&sb, start_index);
     78 		stream_append_u64(&sb, it);
     79 		stream_append_byte(&sb, 0);
     80 		os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1);
     81 		if InvalidHandle(os_w32_shared_memory_semaphores[it])
     82 			break;
     83 		reserved_count++;
     84 	}
     85 
     86 	b32 result = reserved_count == countof(os_w32_shared_memory_semaphores);
     87 	if (!result) {
     88 		for (u32 i = 0; i < reserved_count; i++)
     89 			CloseHandle(os_w32_shared_memory_semaphores[i].value[0]);
     90 	}
     91 
     92 	return result;
     93 }
     94 
     95 function str8
     96 os_open_shared_memory_area(char *name)
     97 {
     98 	struct alignas(16) {
     99 		void *BaseAddress;
    100 		void *AllocationBase;
    101 		u32   AllocationProtect;
    102 		u32   __alignment1;
    103 		u64   RegionSize;
    104 		u32   State;
    105 		u32   Protect;
    106 		u32   Type;
    107 		u32   __alignment2;
    108 	} memory_basic_info;
    109 
    110 	str8 result = {0};
    111 	iptr h = OpenFileMappingA(FILE_MAP_ALL_ACCESS, 0, name);
    112 	if (h != INVALID_FILE) {
    113 		// NOTE(rnp): a size of 0 maps the whole region, we can determine its size after
    114 		void *new = MapViewOfFile(h, FILE_MAP_ALL_ACCESS, 0, 0, 0);
    115 		if (new &&
    116 		    VirtualQuery(new, &memory_basic_info, sizeof(memory_basic_info)) == sizeof(memory_basic_info) &&
    117 		    os_reserve_region_locks())
    118 		{
    119 			result.data   = new;
    120 			result.length = (i64)memory_basic_info.RegionSize;
    121 		}
    122 
    123 		if (new && !result.data)
    124 			UnmapViewOfFile(new);
    125 
    126 		CloseHandle(h);
    127 	}
    128 	return result;
    129 }
    130 
    131 function void
    132 os_close_shared_memory_area(void *memory, i64 size)
    133 {
    134 	UnmapViewOfFile(memory);
    135 }
    136 
    137 #endif
    138 
    139 #define lib_error_check(c, e) lib_error_check_(c, BeamformerLibErrorKind_##e)
    140 function b32
    141 lib_error_check_(b32 condition, BeamformerLibErrorKind error_kind)
    142 {
    143 	b32 result = condition;
    144 	if (!result) g_beamformer_library_context.last_error = error_kind;
    145 	assert(result);
    146 	return result;
    147 }
    148 
    149 function b32
    150 check_shared_memory(void)
    151 {
    152 	b32 result = g_beamformer_library_context.bp != 0;
    153 	if unlikely(!g_beamformer_library_context.bp) {
    154 		str8 shared_memory = os_open_shared_memory_area(OS_SHARED_MEMORY_NAME);
    155 		if (lib_error_check(shared_memory.data != 0, SharedMemory)) {
    156 			BeamformerSharedMemory *bp = (BeamformerSharedMemory *)shared_memory.data;
    157 			result = lib_error_check(bp->version == BEAMFORMER_SHARED_MEMORY_VERSION, VersionMismatch);
    158 			if (result) {
    159 				g_beamformer_library_context.bp = bp;
    160 			} else {
    161 				os_close_shared_memory_area(shared_memory.data, shared_memory.length);
    162 			}
    163 		}
    164 	}
    165 
    166 	if likely(g_beamformer_library_context.bp)
    167 		result = lib_error_check(likely(!g_beamformer_library_context.bp->invalid), InvalidAccess);
    168 	return result;
    169 }
    170 
    171 function b32
    172 valid_parameter_block(u32 block)
    173 {
    174 	b32 result = check_shared_memory();
    175 	if (result) {
    176 		result = lib_error_check(block < g_beamformer_library_context.bp->reserved_parameter_blocks,
    177 		                         ParameterBlockUnallocated);
    178 	}
    179 	return result;
    180 }
    181 
    182 function BeamformWork *
    183 try_push_work_queue(void)
    184 {
    185 	BeamformWork *result = beamform_work_queue_push(&g_beamformer_library_context.bp->external_work_queue);
    186 	lib_error_check(result != 0, WorkQueueFull);
    187 	return result;
    188 }
    189 
    190 function b32
    191 lib_try_lock(i32 lock, i32 timeout_ms)
    192 {
    193 	b32 result = beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, (u32)timeout_ms);
    194 	lib_error_check(result, SyncVariable);
    195 	return result;
    196 }
    197 
    198 function void
    199 lib_release_lock(i32 lock)
    200 {
    201 	beamformer_shared_memory_release_lock(g_beamformer_library_context.bp, lock);
    202 }
    203 
    204 u32
    205 beamformer_get_api_version(void)
    206 {
    207 	return BEAMFORMER_SHARED_MEMORY_VERSION;
    208 }
    209 
    210 const char *
    211 beamformer_error_string(BeamformerLibErrorKind kind)
    212 {
    213 	#define X(type, num, string) string,
    214 	local_persist const char *error_string_table[] = {BEAMFORMER_LIB_ERRORS "invalid error kind"};
    215 	#undef X
    216 	return error_string_table[Min(kind, countof(error_string_table) - 1)];
    217 }
    218 
    219 BeamformerLibErrorKind
    220 beamformer_get_last_error(void)
    221 {
    222 	return g_beamformer_library_context.last_error;
    223 }
    224 
    225 const char *
    226 beamformer_get_last_error_string(void)
    227 {
    228 	return beamformer_error_string(beamformer_get_last_error());
    229 }
    230 
    231 void
    232 beamformer_set_global_timeout(u32 timeout_ms)
    233 {
    234 	g_beamformer_library_context.timeout_ms = timeout_ms;
    235 }
    236 
    237 b32
    238 beamformer_reserve_parameter_blocks(uint32_t count)
    239 {
    240 	b32 result = 0;
    241 	if (check_shared_memory() &&
    242 	    lib_error_check(count <= BeamformerMaxParameterBlocks, ParameterBlockOverflow))
    243 	{
    244 		g_beamformer_library_context.bp->reserved_parameter_blocks = count;
    245 		result = 1;
    246 	}
    247 	return result;
    248 }
    249 
    250 function b32
    251 validate_parameters(BeamformerParameters *bp)
    252 {
    253 	if (!lib_error_check(Between(bp->contrast_mode, 0, BeamformerContrastMode_Count - 1), InvalidContrastMode))
    254 		return 0;
    255 
    256 	u32 contrast_raw_sample_count = bp->acquisition_count * bp->sample_count * beamformer_contrast_mode_samples[bp->contrast_mode];
    257 	if (!lib_error_check(contrast_raw_sample_count <= bp->raw_data_dimensions.x, DataSizeMismatch))
    258 		return 0;
    259 
    260 	// NOTE(rnp): frame size checks
    261 	{
    262 		// TODO(rnp): this check is overly conservative, what if we are exporting something smaller than Float32Complex
    263 		u64 buffer_size     = g_beamformer_library_context.bp->beamformed_frame_buffer_size;
    264 		u64 frame_size      = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z)
    265 		                      * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex];
    266 		u64 incoherent_size = frame_size / 2;
    267 		if (bp->coherency_weighting)
    268 			buffer_size -= incoherent_size;
    269 
    270 		if (!lib_error_check(frame_size <= buffer_size, FrameSizeOverflow))
    271 			return 0;
    272 	}
    273 
    274 	return 1;
    275 }
    276 
    277 function b32
    278 validate_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind)
    279 {
    280 	b32 data_kind_test = Between(data_kind, 0, BeamformerDataKind_Count - 1);
    281 	if (!lib_error_check(data_kind_test, InvalidDataKind))
    282 		return 0;
    283 
    284 	if (!lib_error_check(shader_count <= BeamformerMaxComputeShaderStages, ComputeStageOverflow))
    285 		return 0;
    286 
    287 	for (u32 i = 0; i < shader_count; i++) {
    288 		b32 stage_test = Between(shaders[i], BeamformerShaderKind_ComputeFirst, BeamformerShaderKind_ComputeLast);
    289 		if (!lib_error_check(stage_test, InvalidComputeStage))
    290 			return 0;
    291 
    292 		if (shaders[i] == BeamformerShaderKind_Hilbert &&
    293 		    !lib_error_check(g_beamformer_library_context.bp->capabilities.hilbert != 0, InvalidComputeStage))
    294 			return 0;
    295 
    296 		if (shaders[i] == BeamformerShaderKind_Demodulate &&
    297 		    !lib_error_check(!beamformer_data_kind_complex[data_kind], InvalidDemodulationDataKind))
    298 		{
    299 			return 0;
    300 		}
    301 	}
    302 
    303 	b32 start_stage_test = shaders[0] == BeamformerShaderKind_Demodulate ||
    304 	                       shaders[0] == BeamformerShaderKind_Decode;
    305 	if (!lib_error_check(start_stage_test, InvalidStartShader))
    306 		return 0;
    307 
    308 	return 1;
    309 }
    310 
    311 u64
    312 beamformer_maximum_rf_data_size(void)
    313 {
    314 	u64 result = U64_MAX;
    315 	if (check_shared_memory()) {
    316 		u64 transient_size = g_beamformer_library_context.bp->transient.page_count
    317 		                     * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE;
    318 		result = Min(transient_size, g_beamformer_library_context.bp->capabilities.max_rf_data_size);
    319 	}
    320 	return result;
    321 }
    322 
    323 u64
    324 beamformer_maximum_frames_for_parameters(BeamformerParameters *bp)
    325 {
    326 	u64 result = U64_MAX;
    327 	if (check_shared_memory() && validate_parameters(bp)) {
    328 		// TODO(rnp): overly conservative frame size check
    329 		u64 buffer_size     = g_beamformer_library_context.bp->beamformed_frame_buffer_size;
    330 		u64 frame_size      = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z)
    331 		                      * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex];
    332 		u64 incoherent_size = frame_size / 2;
    333 		if (bp->coherency_weighting)
    334 			buffer_size -= incoherent_size;
    335 		result = buffer_size / frame_size;
    336 	}
    337 	return result;
    338 }
    339 
    340 u64
    341 beamformer_maximum_frames_for_simple_parameters(BeamformerSimpleParameters *bp)
    342 {
    343 	u64 result = beamformer_maximum_frames_for_parameters((BeamformerParameters *)bp);
    344 	return result;
    345 }
    346 
    347 function b32
    348 parameter_block_region_upload(const void *data, u32 size, u32 block, u32 block_offset, i32 timeout_ms)
    349 {
    350 	i32 lock   = BeamformerSharedMemoryLockKind_Count + (i32)block;
    351 	b32 result = valid_parameter_block(block) && lib_try_lock(lock, timeout_ms);
    352 	if (result) {
    353 		memory_copy((u8 *)beamformer_parameter_block(g_beamformer_library_context.bp, block) + block_offset,
    354 		            data, size);
    355 		beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters);
    356 		lib_release_lock(lock);
    357 	}
    358 	return result;
    359 }
    360 
    361 b32
    362 beamformer_set_pipeline_stage_parameters_at(u32 stage_index, i32 parameter, u32 block)
    363 {
    364 	u32 offset  = BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_ComputePipeline];
    365 	offset     += offsetof(BeamformerComputePipeline, parameters);
    366 	offset     += (stage_index % BeamformerMaxComputeShaderStages) * sizeof(BeamformerShaderParameters);
    367 	b32 result  = parameter_block_region_upload(&parameter, sizeof(BeamformerShaderParameters), block,
    368 	                                            offset, g_beamformer_library_context.timeout_ms);
    369 	return result;
    370 }
    371 
    372 b32
    373 beamformer_set_pipeline_stage_parameters(u32 stage_index, i32 parameter)
    374 {
    375 	b32 result = beamformer_set_pipeline_stage_parameters_at(stage_index, parameter, 0);
    376 	return result;
    377 }
    378 
    379 b32
    380 beamformer_push_pipeline_at(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind, u32 block)
    381 {
    382 	b32 result = 0;
    383 	if (check_shared_memory() && validate_pipeline(shaders, shader_count, data_kind)) {
    384 		i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block;
    385 		if (valid_parameter_block(block) && lib_try_lock(lock, g_beamformer_library_context.timeout_ms)) {
    386 			BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block);
    387 			memory_copy(&b->pipeline.shaders, shaders, shader_count * sizeof(*shaders));
    388 			beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters);
    389 			b->pipeline.shader_count = shader_count;
    390 			b->pipeline.data_kind    = data_kind;
    391 			lib_release_lock(lock);
    392 			result = 1;
    393 		}
    394 	}
    395 	return result;
    396 }
    397 
    398 b32
    399 beamformer_push_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind)
    400 {
    401 	b32 result = beamformer_push_pipeline_at(shaders, shader_count, data_kind, 0);
    402 	return result;
    403 }
    404 
    405 b32
    406 beamformer_create_filter(BeamformerFilterParameters *filter, u8 filter_slot, u8 parameter_block)
    407 {
    408 	b32 result = 0;
    409 	if (lib_error_check(filter->kind >= 0 && filter->kind < BeamformerFilterKind_Count, InvalidFilterKind)) {
    410 		if (check_shared_memory()) {
    411 			BeamformWork *work = try_push_work_queue();
    412 			if (work) {
    413 				BeamformerCreateFilterContext *ctx = &work->create_filter_context;
    414 				work->kind = BeamformerWorkKind_CreateFilter;
    415 				ctx->parameters      = *filter;
    416 				ctx->filter_slot     = filter_slot     % BeamformerFilterSlots;
    417 				ctx->parameter_block = parameter_block % BeamformerMaxParameterBlocks;
    418 				beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue);
    419 				result = 1;
    420 			}
    421 		}
    422 	}
    423 	return result;
    424 }
    425 
    426 function void
    427 beamformer_flush_commands(void)
    428 {
    429 	i32 lock = BeamformerSharedMemoryLockKind_DispatchCompute;
    430 	beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, 0);
    431 }
    432 
    433 #define BEAMFORMER_UPLOAD_FNS \
    434 	X(channel_mapping,               i16, 1, ChannelMapping) \
    435 	X(focal_vectors,                 f32, 2, FocalVectors)   \
    436 	X(sparse_elements,               i16, 1, SparseElements) \
    437 	X(transmit_receive_orientations, u8,  1, TransmitReceiveOrientations)
    438 
    439 #define X(name, dtype, elements, region_name) \
    440 b32 beamformer_push_##name ##_at(const dtype *data, u32 count, u32 block) { \
    441 	b32 result = 0; \
    442 	if (lib_error_check(count <= countof(((BeamformerParameterBlock *)0)->name), BufferOverflow)) { \
    443 		result = parameter_block_region_upload(data, count * elements * sizeof(dtype), block, \
    444 		                                       offsetof(BeamformerParameterBlock, name),      \
    445 		                                       g_beamformer_library_context.timeout_ms);      \
    446 	} \
    447 	return result; \
    448 }
    449 BEAMFORMER_UPLOAD_FNS
    450 #undef X
    451 
    452 #define X(name, dtype, ...) \
    453 b32 beamformer_push_##name (const dtype *data, u32 count) { \
    454 	b32 result = beamformer_push_##name ##_at(data, count, 0); \
    455 	return result; \
    456 }
    457 BEAMFORMER_UPLOAD_FNS
    458 #undef X
    459 
    460 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(name) void name(void *restrict output_v, \
    461                                                            void *restrict input_v, \
    462                                                            u32 sample_count)
    463 typedef BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_fn);
    464 
    465 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST \
    466 	X(i16) \
    467 	X(f32) \
    468 	X(f16) \
    469 
    470 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), "");
    471 
    472 #define X(type, ...) \
    473 function BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_##type) \
    474 { \
    475 	type *input_a = (type *)input_v + 0 * sample_count; \
    476 	type *input_b = (type *)input_v + 1 * sample_count; \
    477 	type *input_c = (type *)input_v + 2 * sample_count; \
    478 	type *output  = (type *)output_v; \
    479 	for (u32 sample = 0; sample < sample_count; sample++) \
    480 		output[sample] = input_a[sample] - input_b[sample] - input_c[sample]; \
    481 }
    482 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST
    483 #undef X
    484 
    485 function b32
    486 beamformer_push_data_base(void *data, u64 data_size, i32 timeout_ms, u32 block)
    487 {
    488 	b32 result = 0;
    489 	BeamformerSharedMemory   *sm = g_beamformer_library_context.bp;
    490 	BeamformerParameterBlock *b  = beamformer_parameter_block(sm, block);
    491 	BeamformerParameters     *bp = &b->parameters;
    492 	BeamformerDataKind     data_kind     = b->pipeline.data_kind;
    493 	BeamformerContrastMode contrast_mode = bp->contrast_mode;
    494 
    495 	u64 max_rf_size = sm->capabilities.max_rf_data_size;
    496 	u32 rf_size     = bp->acquisition_count * bp->sample_count * bp->channel_count * beamformer_data_kind_byte_size[data_kind];
    497 	u32 raw_size    = bp->raw_data_dimensions.x * bp->raw_data_dimensions.y * beamformer_data_kind_byte_size[data_kind];
    498 
    499 	// TODO(rnp): support multi push upload so that max_rf_size is actual limit
    500 	if (lib_try_lock(BeamformerSharedMemoryLockKind_UploadRF, timeout_ms)) {
    501 		BeamformerSharedMemoryAllocationHandle rf_handle = beamformer_shared_memory_transient_alloc(sm, .size = rf_size, .align = BEAMFORMER_RF_ALIGN);
    502 		if (lib_error_check(rf_handle != 0, TransientSpaceOverflow) &&
    503 		    lib_error_check(rf_size <= max_rf_size, RFDataSizeOverflow) &&
    504 		    lib_error_check(rf_size <= data_size && data_size == raw_size, DataSizeMismatch))
    505 		{
    506 			u32 channel_count      = bp->channel_count;
    507 			u32 out_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->sample_count * bp->acquisition_count;
    508 			u32 in_channel_stride  = beamformer_data_kind_byte_size[data_kind] * bp->raw_data_dimensions.x;
    509 
    510 			u8 *base = beamformer_shared_memory_allocation_pointer(sm, rf_handle);
    511 			for (u32 channel = 0; channel < channel_count; channel++) {
    512 				u16 data_channel = (u16)b->channel_mapping[channel];
    513 				u32 out_off = out_channel_stride * channel;
    514 				u32 in_off  = in_channel_stride  * data_channel;
    515 
    516 				u8 *memory = base + out_off;
    517 				switch (contrast_mode) {
    518 				default:{
    519 					/* NOTE(rnp): non temporal copy would be better, but we can't ensure
    520 					 * 64 byte boundaries. */
    521 					memory_copy(memory, (u8 *)data + in_off, out_channel_stride);
    522 				}break;
    523 
    524 				case BeamformerContrastMode_A1S2:{
    525 					read_only u8 reduce_a1s2_index_map[] = {
    526 						[BeamformerDataKind_Int16]          = 0,
    527 						[BeamformerDataKind_Int16Complex]   = 0,
    528 						[BeamformerDataKind_Float32]        = 1,
    529 						[BeamformerDataKind_Float32Complex] = 1,
    530 						[BeamformerDataKind_Float16]        = 2,
    531 						[BeamformerDataKind_Float16Complex] = 2,
    532 					};
    533 					static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), "");
    534 
    535 					local_persist beamformer_reduce_a1s2_contrast_fn *reduce_a1s2_fn_table[] = {
    536 						#define X(type, ...) beamformer_reduce_a1s2_contrast_##type,
    537 						BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST
    538 						#undef X
    539 					};
    540 					beamformer_reduce_a1s2_contrast_fn *reduce = reduce_a1s2_fn_table[reduce_a1s2_index_map[data_kind]];
    541 
    542 					u32 sample_count       = bp->sample_count * beamformer_data_kind_element_count[data_kind];
    543 					u32 acquisition_stride = sample_count * beamformer_data_kind_element_size[data_kind];
    544 					for (u32 acquisition = 0; acquisition < bp->acquisition_count; acquisition++) {
    545 						void *out = memory + acquisition * acquisition_stride;
    546 						void *in  = (u8 *)data + in_off + 3 * acquisition * acquisition_stride;
    547 						reduce(out, in, sample_count);
    548 					}
    549 				}break;
    550 				}
    551 			}
    552 
    553 			/* TODO(rnp): need a better way to communicate this */
    554 			atomic_store_u64(&sm->rf_handle, rf_handle);
    555 			g_beamformer_library_context.new_rf = 1;
    556 			result = 1;
    557 		}
    558 
    559 		if (!result && rf_handle) {
    560 			beamformer_shared_memory_allocation_release(sm, rf_handle);
    561 			lib_release_lock(BeamformerSharedMemoryLockKind_UploadRF);
    562 		}
    563 	}
    564 	return result;
    565 }
    566 
    567 function b32
    568 beamformer_start_compute(u64 offset, u32 image_plane_tag, u32 parameter_slot)
    569 {
    570 	b32 result = 0;
    571 	if (check_shared_memory()) {
    572 		u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks;
    573 		if (lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) &&
    574 		    lib_error_check(image_plane_tag < BeamformerViewPlaneTag_Count, InvalidImagePlane))
    575 		{
    576 			// TODO(rnp): this stuff will be passed in rather than taken from the existing set
    577 			BeamformerParameterBlock *pb = beamformer_parameter_block(g_beamformer_library_context.bp, parameter_slot);
    578 			BeamformerParameters     *bp = &pb->parameters;
    579 
    580 			BeamformWork *work = try_push_work_queue();
    581 			if (work) {
    582 				b32 new_rf = g_beamformer_library_context.new_rf;
    583 				g_beamformer_library_context.new_rf = 0;
    584 				work->kind = new_rf ? BeamformerWorkKind_WaitThenCompute : BeamformerWorkKind_Compute;
    585 				work->compute_context.view_plane        = image_plane_tag;
    586 				work->compute_context.parameter_set     = parameter_slot;
    587 				work->compute_context.rf_offset         = offset;
    588 				work->compute_context.steering_angle    = bp->single_focus ? bp->focal_vector.x : pb->focal_vectors[0].x;
    589 				work->compute_context.focus_depth       = bp->single_focus ? bp->focal_vector.y : pb->focal_vectors[0].y;
    590 				work->compute_context.tx_rx_orientation = bp->single_orientation ? bp->transmit_receive_orientation
    591 				                                                                 : pb->transmit_receive_orientations[0];
    592 				memory_copy(&work->compute_context.voxel_transform, &bp->das_voxel_transform, sizeof(bp->das_voxel_transform));
    593 				beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue);
    594 				beamformer_flush_commands();
    595 				result = 1;
    596 			}
    597 		}
    598 	}
    599 	return result;
    600 }
    601 
    602 function b32
    603 beamformer_push_data(void *data, u64 data_size, u32 parameter_slot)
    604 {
    605 	b32 result = 0;
    606 	if (check_shared_memory()) {
    607 		u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks;
    608 		result = lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) &&
    609 		         beamformer_push_data_base(data, data_size, g_beamformer_library_context.timeout_ms, parameter_slot);
    610 	}
    611 	return result;
    612 }
    613 
    614 b32
    615 beamformer_push_data_with_compute(void *data, u32 data_size, u32 image_plane_tag, u32 parameter_slot)
    616 {
    617 	b32 result = beamformer_push_data(data, data_size, parameter_slot) &&
    618 	             beamformer_start_compute(0, image_plane_tag, parameter_slot);
    619 	return result;
    620 }
    621 
    622 b32
    623 beamformer_push_parameters_at(BeamformerParameters *bp, u32 block)
    624 {
    625 	b32 result = check_shared_memory() && validate_parameters(bp);
    626 	if (result) {
    627 		result = parameter_block_region_upload(bp, sizeof(*bp), block,
    628 		                                       offsetof(BeamformerParameterBlock, parameters),
    629 		                                       g_beamformer_library_context.timeout_ms);
    630 		if (result) {
    631 			beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block,
    632 			                                    BeamformerParameterDirtyFlag_NotifyUI);
    633 		}
    634 	}
    635 	return result;
    636 }
    637 
    638 b32
    639 beamformer_push_parameters(BeamformerParameters *bp)
    640 {
    641 	b32 result = beamformer_push_parameters_at(bp, 0);
    642 	return result;
    643 }
    644 
    645 b32
    646 beamformer_push_simple_parameters_at(BeamformerSimpleParameters *bp, u32 block)
    647 {
    648 	b32 result = check_shared_memory();
    649 	if (result) {
    650 		alignas(64) v2 focal_vectors[countof(bp->steering_angles)];
    651 		for (u32 i = 0; i < countof(bp->steering_angles); i++)
    652 			focal_vectors[i] = (v2){{bp->steering_angles[i], bp->focal_depths[i]}};
    653 
    654 		result &= beamformer_push_parameters_at((BeamformerParameters *)bp, block);
    655 		result &= beamformer_push_pipeline_at(bp->compute_stages, bp->compute_stages_count, (BeamformerDataKind)bp->data_kind, block);
    656 		result &= beamformer_push_channel_mapping_at(bp->channel_mapping, bp->channel_count, block);
    657 		result &= beamformer_push_focal_vectors_at((f32 *)focal_vectors, countof(focal_vectors), block);
    658 		result &= beamformer_push_transmit_receive_orientations_at(bp->transmit_receive_orientations,
    659 		                                                           bp->acquisition_count, block);
    660 
    661 		if (bp->acquisition_kind == BeamformerAcquisitionKind_UFORCES ||
    662 		    bp->acquisition_kind == BeamformerAcquisitionKind_UHERCULES)
    663 		{
    664 			result &= beamformer_push_sparse_elements_at(bp->sparse_elements, bp->acquisition_count, block);
    665 		}
    666 
    667 		for (u32 stage = 0; stage < bp->compute_stages_count; stage++)
    668 			result &= beamformer_set_pipeline_stage_parameters_at(stage, bp->compute_stage_parameters[stage], block);
    669 	}
    670 	return result;
    671 }
    672 
    673 b32
    674 beamformer_push_simple_parameters(BeamformerSimpleParameters *bp)
    675 {
    676 	b32 result = beamformer_push_simple_parameters_at(bp, 0);
    677 	return result;
    678 }
    679 
    680 function b32
    681 beamformer_export_buffer(BeamformerExportContext export_context)
    682 {
    683 	BeamformWork *work = try_push_work_queue();
    684 	b32 result = work && lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, 0);
    685 	if (result) {
    686 		work->export_context = export_context;
    687 		work->kind = BeamformerWorkKind_ExportBuffer;
    688 		beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue);
    689 	}
    690 	return result;
    691 }
    692 
    693 function b32
    694 beamformer_export(BeamformerExportContext export, void *out, i32 timeout_ms)
    695 {
    696 	b32 result = 0;
    697 	if (beamformer_export_buffer(export)) {
    698 		/* NOTE(rnp): if this fails it just means that the work from push_data hasn't
    699 		 * started yet. This is here to catch the other case where the work started
    700 		 * and finished before we finished queuing the export work item */
    701 		beamformer_flush_commands();
    702 
    703 		if (lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, timeout_ms)) {
    704 			BeamformerSharedMemory *sm = g_beamformer_library_context.bp;
    705 			memory_copy(out, beamformer_shared_memory_allocation_pointer(sm, export.memory),
    706 			            beamformer_shared_memory_allocation_size(sm, export.memory));
    707 			lib_release_lock(BeamformerSharedMemoryLockKind_ExportSync);
    708 			result = 1;
    709 		}
    710 	}
    711 	return result;
    712 }
    713 
    714 BEAMFORMER_LIB_EXPORT b32
    715 beamformer_get_last_frames(void *out_data, u64 out_data_size, u32 count)
    716 {
    717 	BeamformerExportContext export = {0};
    718 	export.kind  = BeamformerExportKind_BeamformedData;
    719 	export.count = count;
    720 	if (out_data_size) {
    721 		export.memory = beamformer_shared_memory_transient_alloc(g_beamformer_library_context.bp,
    722 		                                                         .size = out_data_size, .align = 64);
    723 	}
    724 
    725 	b32 result = lib_error_check(export.memory != 0, TransientSpaceOverflow) && count;
    726 	if (result)
    727 		result = beamformer_export(export, out_data, g_beamformer_library_context.timeout_ms);
    728 	if (export.memory)
    729 		beamformer_shared_memory_allocation_release(g_beamformer_library_context.bp, export.memory);
    730 
    731 	return result;
    732 }
    733 
    734 b32
    735 beamformer_beamform_data(BeamformerSimpleParameters *bp, void *data, uint32_t data_size,
    736                          void *out_data, int32_t timeout_ms)
    737 {
    738 	b32 result = beamformer_push_simple_parameters(bp);
    739 	if (result) {
    740 		beamformer_set_global_timeout(timeout_ms);
    741 
    742 		iv3 output_points = bp->output_points.xyz;
    743 		output_points.E[0] = Max(1, output_points.E[0]);
    744 		output_points.E[1] = Max(1, output_points.E[1]);
    745 		output_points.E[2] = Max(1, output_points.E[2]);
    746 
    747 		b32 complex = 0;
    748 		for (u32 stage = 0; stage < bp->compute_stages_count; stage++) {
    749 			BeamformerShaderKind shader = (BeamformerShaderKind)bp->compute_stages[stage];
    750 			complex |= shader == BeamformerShaderKind_Demodulate || shader == BeamformerShaderKind_Hilbert;
    751 		}
    752 
    753 		u64 output_size = output_points.x * output_points.y * output_points.z * sizeof(f32);
    754 		if (complex) output_size *= 2;
    755 
    756 		result = beamformer_push_data_with_compute(data, data_size, 0, 0);
    757 
    758 		if (result && out_data)
    759 			result = beamformer_get_last_frames(out_data, output_size, 1);
    760 	}
    761 	return result;
    762 }
    763 
    764 function b32
    765 beamformer_compute_timings(BeamformerComputeStatsTable *output, i32 timeout_ms)
    766 {
    767 	b32 result = 0;
    768 	if (check_shared_memory()) {
    769 		BeamformerExportContext export = {0};
    770 		export.kind   = BeamformerExportKind_Stats;
    771 		export.memory = beamformer_shared_memory_transient_alloc(g_beamformer_library_context.bp,
    772 		                                                         .size  = sizeof(BeamformerComputeStatsTable),
    773 		                                                         .align = alignof(BeamformerComputeStatsTable));
    774 		if (lib_error_check(export.memory != 0, TransientSpaceOverflow)) {
    775 			result = beamformer_export(export, output, timeout_ms);
    776 			beamformer_shared_memory_allocation_release(g_beamformer_library_context.bp, export.memory);
    777 		}
    778 	}
    779 	return result;
    780 }
    781 
    782 i32
    783 beamformer_live_parameters_get_dirty_flag(void)
    784 {
    785 	i32 result = -1;
    786 	if (check_shared_memory()) {
    787 		u32 flag = ctz_u64(g_beamformer_library_context.bp->live_imaging_dirty_flags);
    788 		if (flag != 64) {
    789 			atomic_and_u32(&g_beamformer_library_context.bp->live_imaging_dirty_flags, ~(1u << flag));
    790 			result = (i32)flag;
    791 		}
    792 	}
    793 	return result;
    794 }
    795 
    796 BeamformerLiveImagingParameters *
    797 beamformer_get_live_parameters(void)
    798 {
    799 	BeamformerLiveImagingParameters *result = 0;
    800 	if (check_shared_memory()) result = &g_beamformer_library_context.bp->live_imaging_parameters;
    801 	return result;
    802 }
    803 
    804 b32
    805 beamformer_set_live_parameters(BeamformerLiveImagingParameters *new)
    806 {
    807 	b32 result = 0;
    808 	if (check_shared_memory()) {
    809 		memory_copy(&g_beamformer_library_context.bp->live_imaging_parameters, new, sizeof(*new));
    810 		store_fence();
    811 		result = 1;
    812 	}
    813 	return result;
    814 }