ogl_beamformer_lib.c (26679B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_IMPORT static 3 4 #ifndef BASE_PLATFORM_H 5 #define BASE_PLATFORM_NO_MAIN 1 6 #define BASE_EXPORT static 7 #endif 8 9 #include "../util.h" 10 11 #include "../generated/beamformer.c" 12 #include "ogl_beamformer_lib_base.h" 13 14 #if OS_LINUX 15 #include "../base_linux.c" 16 #elif OS_WINDOWS 17 #include "../base_win32.c" 18 19 W32(iptr) OpenFileMappingA(u32, b32, c8 *); 20 21 #else 22 #error Unsupported Platform 23 #endif 24 25 #include "../util_os.c" 26 #include "../beamformer_compute_stats.c" 27 #include "../beamformer_shared_memory.c" 28 29 global struct { 30 BeamformerSharedMemory *bp; 31 i32 timeout_ms; 32 BeamformerLibErrorKind last_error; 33 i64 shared_memory_size; 34 } g_beamformer_library_context; 35 36 #if OS_LINUX 37 38 function str8 39 os_open_shared_memory_area(char *name) 40 { 41 str8 result = {0}; 42 i32 fd = shm_open(name, O_RDWR, S_IRUSR|S_IWUSR); 43 if (fd > 0) { 44 struct stat sb; 45 if (fstat(fd, &sb) != -1) { 46 void *new = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 47 if (new != MAP_FAILED) { 48 result.data = new; 49 result.length = sb.st_size; 50 } 51 } 52 close(fd); 53 } 54 return result; 55 } 56 57 function void 58 os_close_shared_memory_area(void *memory, i64 size) 59 { 60 munmap(memory, size); 61 } 62 63 #elif OS_WINDOWS 64 65 W32(u64) VirtualQuery(void *base_address, void *memory_basic_info, u64 memory_basic_info_size); 66 W32(b32) UnmapViewOfFile(void *); 67 68 function b32 69 os_reserve_region_locks(void) 70 { 71 u8 buffer[1024]; 72 Stream sb = {.data = buffer, .cap = countof(buffer)}; 73 stream_append_str8(&sb, str8(OS_SHARED_MEMORY_NAME "_lock_")); 74 75 i32 start_index = sb.widx; 76 u32 reserved_count = 0; 77 for EachElement(os_w32_shared_memory_semaphores, it) { 78 stream_reset(&sb, start_index); 79 stream_append_u64(&sb, it); 80 stream_append_byte(&sb, 0); 81 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 82 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 83 break; 84 reserved_count++; 85 } 86 87 b32 result = reserved_count == countof(os_w32_shared_memory_semaphores); 88 if (!result) { 89 for (u32 i = 0; i < reserved_count; i++) 90 CloseHandle(os_w32_shared_memory_semaphores[i].value[0]); 91 } 92 93 return result; 94 } 95 96 function str8 97 os_open_shared_memory_area(char *name) 98 { 99 struct alignas(16) { 100 void *BaseAddress; 101 void *AllocationBase; 102 u32 AllocationProtect; 103 u32 __alignment1; 104 u64 RegionSize; 105 u32 State; 106 u32 Protect; 107 u32 Type; 108 u32 __alignment2; 109 } memory_basic_info; 110 111 str8 result = {0}; 112 iptr h = OpenFileMappingA(FILE_MAP_ALL_ACCESS, 0, name); 113 if (h != INVALID_FILE) { 114 // NOTE(rnp): a size of 0 maps the whole region, we can determine its size after 115 void *new = MapViewOfFile(h, FILE_MAP_ALL_ACCESS, 0, 0, 0); 116 if (new && 117 VirtualQuery(new, &memory_basic_info, sizeof(memory_basic_info)) == sizeof(memory_basic_info) && 118 os_reserve_region_locks()) 119 { 120 result.data = new; 121 result.length = (i64)memory_basic_info.RegionSize; 122 } 123 124 if (new && !result.data) 125 UnmapViewOfFile(new); 126 127 CloseHandle(h); 128 } 129 return result; 130 } 131 132 function void 133 os_close_shared_memory_area(void *memory, i64 size) 134 { 135 UnmapViewOfFile(memory); 136 } 137 138 #endif 139 140 #define lib_error_check(c, e) lib_error_check_(c, BeamformerLibErrorKind_##e) 141 function b32 142 lib_error_check_(b32 condition, BeamformerLibErrorKind error_kind) 143 { 144 b32 result = condition; 145 if (!result) g_beamformer_library_context.last_error = error_kind; 146 assert(result); 147 return result; 148 } 149 150 function b32 151 check_shared_memory(void) 152 { 153 b32 result = g_beamformer_library_context.bp != 0; 154 if unlikely(!g_beamformer_library_context.bp) { 155 str8 shared_memory = os_open_shared_memory_area(OS_SHARED_MEMORY_NAME); 156 if (lib_error_check(shared_memory.data != 0, SharedMemory)) { 157 BeamformerSharedMemory *bp = (BeamformerSharedMemory *)shared_memory.data; 158 result = lib_error_check(bp->version == BEAMFORMER_SHARED_MEMORY_VERSION, VersionMismatch); 159 if (result) { 160 g_beamformer_library_context.bp = bp; 161 g_beamformer_library_context.shared_memory_size = shared_memory.length; 162 } else { 163 os_close_shared_memory_area(shared_memory.data, shared_memory.length); 164 } 165 } 166 } 167 168 if likely(g_beamformer_library_context.bp) 169 result = lib_error_check(likely(!g_beamformer_library_context.bp->invalid), InvalidAccess); 170 return result; 171 } 172 173 function b32 174 valid_parameter_block(u32 block) 175 { 176 b32 result = check_shared_memory(); 177 if (result) { 178 result = lib_error_check(block < g_beamformer_library_context.bp->reserved_parameter_blocks, 179 ParameterBlockUnallocated); 180 } 181 return result; 182 } 183 184 function BeamformWork * 185 try_push_work_queue(void) 186 { 187 BeamformWork *result = beamform_work_queue_push(&g_beamformer_library_context.bp->external_work_queue); 188 lib_error_check(result != 0, WorkQueueFull); 189 return result; 190 } 191 192 function b32 193 lib_try_lock(i32 lock, i32 timeout_ms) 194 { 195 b32 result = beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, (u32)timeout_ms); 196 lib_error_check(result, SyncVariable); 197 return result; 198 } 199 200 function void 201 lib_release_lock(i32 lock) 202 { 203 beamformer_shared_memory_release_lock(g_beamformer_library_context.bp, lock); 204 } 205 206 u32 207 beamformer_get_api_version(void) 208 { 209 return BEAMFORMER_SHARED_MEMORY_VERSION; 210 } 211 212 const char * 213 beamformer_error_string(BeamformerLibErrorKind kind) 214 { 215 #define X(type, num, string) string, 216 local_persist const char *error_string_table[] = {BEAMFORMER_LIB_ERRORS "invalid error kind"}; 217 #undef X 218 return error_string_table[Min(kind, countof(error_string_table) - 1)]; 219 } 220 221 BeamformerLibErrorKind 222 beamformer_get_last_error(void) 223 { 224 return g_beamformer_library_context.last_error; 225 } 226 227 const char * 228 beamformer_get_last_error_string(void) 229 { 230 return beamformer_error_string(beamformer_get_last_error()); 231 } 232 233 void 234 beamformer_set_global_timeout(u32 timeout_ms) 235 { 236 g_beamformer_library_context.timeout_ms = timeout_ms; 237 } 238 239 b32 240 beamformer_reserve_parameter_blocks(uint32_t count) 241 { 242 b32 result = 0; 243 if (check_shared_memory() && 244 lib_error_check(count <= BeamformerMaxParameterBlocks, ParameterBlockOverflow)) 245 { 246 g_beamformer_library_context.bp->reserved_parameter_blocks = count; 247 result = 1; 248 } 249 return result; 250 } 251 252 function b32 253 validate_parameters(BeamformerParameters *bp) 254 { 255 if (!lib_error_check(Between(bp->contrast_mode, 0, BeamformerContrastMode_Count - 1), InvalidContrastMode)) 256 return 0; 257 258 u32 contrast_raw_sample_count = bp->acquisition_count * bp->sample_count * beamformer_contrast_mode_samples[bp->contrast_mode]; 259 if (!lib_error_check(contrast_raw_sample_count <= bp->raw_data_dimensions.x, DataSizeMismatch)) 260 return 0; 261 262 // NOTE(rnp): frame size checks 263 { 264 // TODO(rnp): this check is overly conservative, what if we are exporting something smaller than Float32Complex 265 u64 buffer_size = g_beamformer_library_context.bp->beamformed_frame_buffer_size; 266 u64 frame_size = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z) 267 * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex]; 268 u64 incoherent_size = frame_size / 2; 269 if (bp->coherency_weighting) 270 buffer_size -= incoherent_size; 271 272 if (!lib_error_check(frame_size <= buffer_size, FrameSizeOverflow)) 273 return 0; 274 } 275 276 return 1; 277 } 278 279 function b32 280 validate_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind) 281 { 282 b32 data_kind_test = Between(data_kind, 0, BeamformerDataKind_Count - 1); 283 if (!lib_error_check(data_kind_test, InvalidDataKind)) 284 return 0; 285 286 if (!lib_error_check(shader_count <= BeamformerMaxComputeShaderStages, ComputeStageOverflow)) 287 return 0; 288 289 for (u32 i = 0; i < shader_count; i++) { 290 b32 stage_test = Between(shaders[i], BeamformerShaderKind_ComputeFirst, BeamformerShaderKind_ComputeLast); 291 if (!lib_error_check(stage_test, InvalidComputeStage)) 292 return 0; 293 294 if (shaders[i] == BeamformerShaderKind_Hilbert && 295 !lib_error_check(g_beamformer_library_context.bp->capabilities.hilbert != 0, InvalidComputeStage)) 296 return 0; 297 298 if (shaders[i] == BeamformerShaderKind_Demodulate && 299 !lib_error_check(!beamformer_data_kind_complex[data_kind], InvalidDemodulationDataKind)) 300 { 301 return 0; 302 } 303 } 304 305 b32 start_stage_test = shaders[0] == BeamformerShaderKind_Demodulate || 306 shaders[0] == BeamformerShaderKind_Decode; 307 if (!lib_error_check(start_stage_test, InvalidStartShader)) 308 return 0; 309 310 return 1; 311 } 312 313 u64 314 beamformer_maximum_rf_data_size(void) 315 { 316 u64 result = U64_MAX; 317 if (check_shared_memory()) { 318 Arena *sm = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 319 g_beamformer_library_context.shared_memory_size); 320 result = Min(sm->reserved - sm->position, g_beamformer_library_context.bp->capabilities.max_rf_data_size); 321 } 322 return result; 323 } 324 325 u64 326 beamformer_maximum_frames_for_parameters(BeamformerParameters *bp) 327 { 328 u64 result = U64_MAX; 329 if (check_shared_memory() && validate_parameters(bp)) { 330 // TODO(rnp): overly conservative frame size check 331 u64 buffer_size = g_beamformer_library_context.bp->beamformed_frame_buffer_size; 332 u64 frame_size = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z) 333 * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex]; 334 u64 incoherent_size = frame_size / 2; 335 if (bp->coherency_weighting) 336 buffer_size -= incoherent_size; 337 result = buffer_size / frame_size; 338 } 339 return result; 340 } 341 342 u64 343 beamformer_maximum_frames_for_simple_parameters(BeamformerSimpleParameters *bp) 344 { 345 u64 result = beamformer_maximum_frames_for_parameters((BeamformerParameters *)bp); 346 return result; 347 } 348 349 function b32 350 parameter_block_region_upload(void *data, u32 size, u32 block, u32 block_offset, i32 timeout_ms) 351 { 352 i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block; 353 b32 result = valid_parameter_block(block) && lib_try_lock(lock, timeout_ms); 354 if (result) { 355 memory_copy((u8 *)beamformer_parameter_block(g_beamformer_library_context.bp, block) + block_offset, 356 data, size); 357 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters); 358 lib_release_lock(lock); 359 } 360 return result; 361 } 362 363 b32 364 beamformer_set_pipeline_stage_parameters_at(u32 stage_index, i32 parameter, u32 block) 365 { 366 u32 offset = BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_ComputePipeline]; 367 offset += offsetof(BeamformerComputePipeline, parameters); 368 offset += (stage_index % BeamformerMaxComputeShaderStages) * sizeof(BeamformerShaderParameters); 369 b32 result = parameter_block_region_upload(¶meter, sizeof(BeamformerShaderParameters), block, 370 offset, g_beamformer_library_context.timeout_ms); 371 return result; 372 } 373 374 b32 375 beamformer_set_pipeline_stage_parameters(u32 stage_index, i32 parameter) 376 { 377 b32 result = beamformer_set_pipeline_stage_parameters_at(stage_index, parameter, 0); 378 return result; 379 } 380 381 b32 382 beamformer_push_pipeline_at(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind, u32 block) 383 { 384 b32 result = 0; 385 if (check_shared_memory() && validate_pipeline(shaders, shader_count, data_kind)) { 386 i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block; 387 if (valid_parameter_block(block) && lib_try_lock(lock, g_beamformer_library_context.timeout_ms)) { 388 BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block); 389 memory_copy(&b->pipeline.shaders, shaders, shader_count * sizeof(*shaders)); 390 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters); 391 b->pipeline.shader_count = shader_count; 392 b->pipeline.data_kind = data_kind; 393 lib_release_lock(lock); 394 result = 1; 395 } 396 } 397 return result; 398 } 399 400 b32 401 beamformer_push_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind) 402 { 403 b32 result = beamformer_push_pipeline_at(shaders, shader_count, data_kind, 0); 404 return result; 405 } 406 407 b32 408 beamformer_create_filter(BeamformerFilterParameters *filter, u8 filter_slot, u8 parameter_block) 409 { 410 b32 result = 0; 411 if (lib_error_check(filter->kind >= 0 && filter->kind < BeamformerFilterKind_Count, InvalidFilterKind)) { 412 if (check_shared_memory()) { 413 BeamformWork *work = try_push_work_queue(); 414 if (work) { 415 BeamformerCreateFilterContext *ctx = &work->create_filter_context; 416 work->kind = BeamformerWorkKind_CreateFilter; 417 ctx->parameters = *filter; 418 ctx->filter_slot = filter_slot % BeamformerFilterSlots; 419 ctx->parameter_block = parameter_block % BeamformerMaxParameterBlocks; 420 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 421 result = 1; 422 } 423 } 424 } 425 return result; 426 } 427 428 function void 429 beamformer_flush_commands(void) 430 { 431 i32 lock = BeamformerSharedMemoryLockKind_DispatchCompute; 432 beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, 0); 433 } 434 435 #define BEAMFORMER_UPLOAD_FNS \ 436 X(channel_mapping, i16, 1, ChannelMapping) \ 437 X(focal_vectors, f32, 2, FocalVectors) \ 438 X(sparse_elements, i16, 1, SparseElements) \ 439 X(transmit_receive_orientations, u8, 1, TransmitReceiveOrientations) 440 441 #define X(name, dtype, elements, region_name) \ 442 b32 beamformer_push_##name ##_at(dtype *data, u32 count, u32 block) { \ 443 b32 result = 0; \ 444 if (lib_error_check(count <= countof(((BeamformerParameterBlock *)0)->name), BufferOverflow)) { \ 445 result = parameter_block_region_upload(data, count * elements * sizeof(dtype), block, \ 446 offsetof(BeamformerParameterBlock, name), \ 447 g_beamformer_library_context.timeout_ms); \ 448 } \ 449 return result; \ 450 } 451 BEAMFORMER_UPLOAD_FNS 452 #undef X 453 454 #define X(name, dtype, ...) \ 455 b32 beamformer_push_##name (dtype *data, u32 count) { \ 456 b32 result = beamformer_push_##name ##_at(data, count, 0); \ 457 return result; \ 458 } 459 BEAMFORMER_UPLOAD_FNS 460 #undef X 461 462 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(name) void name(void *restrict output_v, \ 463 void *restrict input_v, \ 464 u32 sample_count) 465 typedef BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_fn); 466 467 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST \ 468 X(i16) \ 469 X(f32) \ 470 X(f16) \ 471 472 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), ""); 473 474 #define X(type, ...) \ 475 function BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_##type) \ 476 { \ 477 type *input_a = (type *)input_v + 0 * sample_count; \ 478 type *input_b = (type *)input_v + 1 * sample_count; \ 479 type *input_c = (type *)input_v + 2 * sample_count; \ 480 type *output = (type *)output_v; \ 481 for (u32 sample = 0; sample < sample_count; sample++) \ 482 output[sample] = input_a[sample] - input_b[sample] - input_c[sample]; \ 483 } 484 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST 485 #undef X 486 487 function b32 488 beamformer_push_data_base(void *data, u32 data_size, i32 timeout_ms, u32 block) 489 { 490 b32 result = 0; 491 Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 492 g_beamformer_library_context.shared_memory_size); 493 BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block); 494 BeamformerParameters *bp = &b->parameters; 495 BeamformerDataKind data_kind = b->pipeline.data_kind; 496 BeamformerContrastMode contrast_mode = bp->contrast_mode; 497 498 499 u64 arena_size = scratch->reserved - scratch->position; 500 u64 max_rf_size = g_beamformer_library_context.bp->capabilities.max_rf_data_size; 501 u32 rf_size = bp->acquisition_count * bp->sample_count * bp->channel_count * beamformer_data_kind_byte_size[data_kind]; 502 u32 raw_size = bp->raw_data_dimensions.x * bp->raw_data_dimensions.y * beamformer_data_kind_byte_size[data_kind]; 503 504 // TODO(rnp): support multi push upload so that max_rf_size is actual limit 505 if (lib_error_check(rf_size <= arena_size, BufferOverflow) && 506 lib_error_check(rf_size <= max_rf_size, RFDataSizeOverflow) && 507 lib_error_check(rf_size <= data_size && data_size == raw_size, DataSizeMismatch)) 508 { 509 if (lib_try_lock(BeamformerSharedMemoryLockKind_UploadRF, timeout_ms)) { 510 if (lib_try_lock(BeamformerSharedMemoryLockKind_ScratchSpace, 0)) { 511 u32 channel_count = bp->channel_count; 512 u32 out_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->sample_count * bp->acquisition_count; 513 u32 in_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->raw_data_dimensions.x; 514 515 for (u32 channel = 0; channel < channel_count; channel++) { 516 u16 data_channel = (u16)b->channel_mapping[channel]; 517 u32 out_off = out_channel_stride * channel; 518 u32 in_off = in_channel_stride * data_channel; 519 u8 *memory = (u8 *)scratch + scratch->position + out_off; 520 switch (contrast_mode) { 521 default:{ 522 /* NOTE(rnp): non temporal copy would be better, but we can't ensure 523 * 64 byte boundaries. */ 524 memory_copy(memory, (u8 *)data + in_off, out_channel_stride); 525 }break; 526 527 case BeamformerContrastMode_A1S2:{ 528 read_only local_persist u8 reduce_a1s2_index_map[] = { 529 [BeamformerDataKind_Int16] = 0, 530 [BeamformerDataKind_Int16Complex] = 0, 531 [BeamformerDataKind_Float32] = 1, 532 [BeamformerDataKind_Float32Complex] = 1, 533 [BeamformerDataKind_Float16] = 2, 534 [BeamformerDataKind_Float16Complex] = 2, 535 }; 536 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), ""); 537 538 read_only local_persist beamformer_reduce_a1s2_contrast_fn *reduce_a1s2_fn_table[] = { 539 #define X(type, ...) beamformer_reduce_a1s2_contrast_##type, 540 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST 541 #undef X 542 }; 543 544 u32 sample_count = bp->sample_count * beamformer_data_kind_element_count[data_kind]; 545 u32 acquisition_stride = sample_count * beamformer_data_kind_element_size[data_kind]; 546 for (u32 acquisition = 0; acquisition < bp->acquisition_count; acquisition++) { 547 void *out = memory + acquisition * acquisition_stride; 548 void *in = (u8 *)data + in_off + 3 * acquisition * acquisition_stride; 549 reduce_a1s2_fn_table[reduce_a1s2_index_map[data_kind]](out, in, sample_count); 550 } 551 }break; 552 } 553 } 554 555 lib_release_lock(BeamformerSharedMemoryLockKind_ScratchSpace); 556 /* TODO(rnp): need a better way to communicate this */ 557 u64 rf_block_rf_size = (u64)block << 32ULL | (u64)rf_size; 558 atomic_store_u64(&g_beamformer_library_context.bp->rf_block_rf_size, rf_block_rf_size); 559 result = 1; 560 } 561 } 562 } 563 return result; 564 } 565 566 b32 567 beamformer_push_data_with_compute(void *data, u32 data_size, u32 image_plane_tag, u32 parameter_slot) 568 { 569 b32 result = 0; 570 if (check_shared_memory()) { 571 u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks; 572 if (lib_error_check(image_plane_tag < BeamformerViewPlaneTag_Count, InvalidImagePlane) && 573 lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) && 574 beamformer_push_data_base(data, data_size, g_beamformer_library_context.timeout_ms, parameter_slot)) 575 { 576 BeamformWork *work = try_push_work_queue(); 577 if (work) { 578 work->kind = BeamformerWorkKind_ComputeIndirect; 579 work->compute_context.view_plane = image_plane_tag; 580 work->compute_context.parameter_block = parameter_slot; 581 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 582 beamformer_flush_commands(); 583 result = 1; 584 } 585 } 586 } 587 return result; 588 } 589 590 b32 591 beamformer_push_parameters_at(BeamformerParameters *bp, u32 block) 592 { 593 b32 result = check_shared_memory() && validate_parameters(bp); 594 if (result) { 595 result = parameter_block_region_upload(bp, sizeof(*bp), block, 596 offsetof(BeamformerParameterBlock, parameters), 597 g_beamformer_library_context.timeout_ms); 598 if (result) { 599 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, 600 BeamformerParameterDirtyFlag_NotifyUI); 601 } 602 } 603 return result; 604 } 605 606 b32 607 beamformer_push_parameters(BeamformerParameters *bp) 608 { 609 b32 result = beamformer_push_parameters_at(bp, 0); 610 return result; 611 } 612 613 b32 614 beamformer_push_simple_parameters_at(BeamformerSimpleParameters *bp, u32 block) 615 { 616 b32 result = check_shared_memory(); 617 if (result) { 618 alignas(64) v2 focal_vectors[countof(bp->steering_angles)]; 619 for (u32 i = 0; i < countof(bp->steering_angles); i++) 620 focal_vectors[i] = (v2){{bp->steering_angles[i], bp->focal_depths[i]}}; 621 622 result &= beamformer_push_parameters_at((BeamformerParameters *)bp, block); 623 result &= beamformer_push_pipeline_at(bp->compute_stages, bp->compute_stages_count, (BeamformerDataKind)bp->data_kind, block); 624 result &= beamformer_push_channel_mapping_at(bp->channel_mapping, bp->channel_count, block); 625 result &= beamformer_push_focal_vectors_at((f32 *)focal_vectors, countof(focal_vectors), block); 626 result &= beamformer_push_transmit_receive_orientations_at(bp->transmit_receive_orientations, 627 bp->acquisition_count, block); 628 629 if (bp->acquisition_kind == BeamformerAcquisitionKind_UFORCES || 630 bp->acquisition_kind == BeamformerAcquisitionKind_UHERCULES) 631 { 632 result &= beamformer_push_sparse_elements_at(bp->sparse_elements, bp->acquisition_count, block); 633 } 634 635 for (u32 stage = 0; stage < bp->compute_stages_count; stage++) 636 result &= beamformer_set_pipeline_stage_parameters_at(stage, bp->compute_stage_parameters[stage], block); 637 } 638 return result; 639 } 640 641 b32 642 beamformer_push_simple_parameters(BeamformerSimpleParameters *bp) 643 { 644 b32 result = beamformer_push_simple_parameters_at(bp, 0); 645 return result; 646 } 647 648 function b32 649 beamformer_export_buffer(BeamformerExportContext export_context) 650 { 651 BeamformWork *work = try_push_work_queue(); 652 b32 result = work && lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, 0); 653 if (result) { 654 work->export_context = export_context; 655 work->kind = BeamformerWorkKind_ExportBuffer; 656 work->lock = BeamformerSharedMemoryLockKind_ScratchSpace; 657 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 658 } 659 return result; 660 } 661 662 function b32 663 beamformer_export(BeamformerExportContext export, void *out, i32 timeout_ms) 664 { 665 b32 result = 0; 666 if (beamformer_export_buffer(export)) { 667 /* NOTE(rnp): if this fails it just means that the work from push_data hasn't 668 * started yet. This is here to catch the other case where the work started 669 * and finished before we finished queuing the export work item */ 670 beamformer_flush_commands(); 671 672 if (lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, timeout_ms)) { 673 if (lib_try_lock(BeamformerSharedMemoryLockKind_ScratchSpace, 0)) { 674 void *sm = beamformer_shared_memory_data_pointer(g_beamformer_library_context.bp, 675 g_beamformer_library_context.shared_memory_size); 676 memory_copy(out, sm, export.size); 677 lib_release_lock(BeamformerSharedMemoryLockKind_ScratchSpace); 678 result = 1; 679 } 680 lib_release_lock(BeamformerSharedMemoryLockKind_ExportSync); 681 } 682 } 683 return result; 684 } 685 686 BEAMFORMER_LIB_EXPORT b32 687 beamformer_get_last_frames(void *out_data, u64 out_data_size, u32 count) 688 { 689 BeamformerExportContext export = {0}; 690 export.kind = BeamformerExportKind_BeamformedData; 691 export.count = count; 692 export.size = out_data_size; 693 b32 result = out_data && out_data_size && count && beamformer_export(export, out_data, g_beamformer_library_context.timeout_ms); 694 return result; 695 } 696 697 b32 698 beamformer_beamform_data(BeamformerSimpleParameters *bp, void *data, uint32_t data_size, 699 void *out_data, int32_t timeout_ms) 700 { 701 b32 result = beamformer_push_simple_parameters(bp); 702 if (result) { 703 iv3 output_points = bp->output_points.xyz; 704 output_points.E[0] = Max(1, output_points.E[0]); 705 output_points.E[1] = Max(1, output_points.E[1]); 706 output_points.E[2] = Max(1, output_points.E[2]); 707 708 b32 complex = 0; 709 for (u32 stage = 0; stage < bp->compute_stages_count; stage++) { 710 BeamformerShaderKind shader = (BeamformerShaderKind)bp->compute_stages[stage]; 711 complex |= shader == BeamformerShaderKind_Demodulate || shader == BeamformerShaderKind_Hilbert; 712 } 713 714 u64 output_size = output_points.x * output_points.y * output_points.z * sizeof(f32); 715 if (complex) output_size *= 2; 716 717 Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 718 g_beamformer_library_context.shared_memory_size); 719 u64 scratch_size = scratch->reserved - scratch->position; 720 if (result && out_data) result &= lib_error_check(output_size <= scratch_size, ExportSpaceOverflow); 721 722 if (result) { 723 result = beamformer_push_data_with_compute(data, data_size, 0, 0); 724 if (result && out_data) 725 result = beamformer_get_last_frames(out_data, output_size, 1); 726 } 727 } 728 return result; 729 } 730 731 BEAMFORMER_LIB_EXPORT b32 732 beamformer_compute_timings(BeamformerComputeStatsTable *output, i32 timeout_ms) 733 { 734 b32 result = 0; 735 if (check_shared_memory()) { 736 Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 737 g_beamformer_library_context.shared_memory_size); 738 u64 scratch_size = scratch->reserved - scratch->position; 739 if (lib_error_check(sizeof(*output) <= scratch_size, ExportSpaceOverflow)) { 740 BeamformerExportContext export = {0}; 741 export.kind = BeamformerExportKind_Stats; 742 export.size = sizeof(*output); 743 result = beamformer_export(export, output, timeout_ms); 744 } 745 } 746 return result; 747 } 748 749 i32 750 beamformer_live_parameters_get_dirty_flag(void) 751 { 752 i32 result = -1; 753 if (check_shared_memory()) { 754 u32 flag = ctz_u64(g_beamformer_library_context.bp->live_imaging_dirty_flags); 755 if (flag != 64) { 756 atomic_and_u32(&g_beamformer_library_context.bp->live_imaging_dirty_flags, ~(1u << flag)); 757 result = (i32)flag; 758 } 759 } 760 return result; 761 } 762 763 BeamformerLiveImagingParameters * 764 beamformer_get_live_parameters(void) 765 { 766 BeamformerLiveImagingParameters *result = 0; 767 if (check_shared_memory()) result = &g_beamformer_library_context.bp->live_imaging_parameters; 768 return result; 769 } 770 771 b32 772 beamformer_set_live_parameters(BeamformerLiveImagingParameters *new) 773 { 774 b32 result = 0; 775 if (check_shared_memory()) { 776 memory_copy(&g_beamformer_library_context.bp->live_imaging_parameters, new, sizeof(*new)); 777 store_fence(); 778 result = 1; 779 } 780 return result; 781 }