ogl_beamformer_lib.c (27550B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_IMPORT static 3 4 #ifndef BASE_PLATFORM_H 5 #define BASE_PLATFORM_NO_MAIN 1 6 #define BASE_EXPORT static 7 #endif 8 9 #include "../util.h" 10 11 #include "../generated/beamformer.c" 12 #include "ogl_beamformer_lib_base.h" 13 14 #if OS_LINUX 15 #include "../base_linux.c" 16 #elif OS_WINDOWS 17 #include "../base_win32.c" 18 19 W32(iptr) OpenFileMappingA(u32, b32, c8 *); 20 21 #else 22 #error Unsupported Platform 23 #endif 24 25 #include "../beamformer_compute_stats.c" 26 #include "../beamformer_shared_memory.c" 27 28 global struct { 29 BeamformerSharedMemory *bp; 30 i32 timeout_ms; 31 BeamformerLibErrorKind last_error; 32 i64 shared_memory_size; 33 } g_beamformer_library_context; 34 35 #if OS_LINUX 36 37 function str8 38 os_open_shared_memory_area(char *name) 39 { 40 str8 result = {0}; 41 i32 fd = shm_open(name, O_RDWR, S_IRUSR|S_IWUSR); 42 if (fd > 0) { 43 struct stat sb; 44 if (fstat(fd, &sb) != -1) { 45 void *new = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 46 if (new != MAP_FAILED) { 47 result.data = new; 48 result.length = sb.st_size; 49 } 50 } 51 close(fd); 52 } 53 return result; 54 } 55 56 function void 57 os_close_shared_memory_area(void *memory, i64 size) 58 { 59 munmap(memory, size); 60 } 61 62 #elif OS_WINDOWS 63 64 W32(u64) VirtualQuery(void *base_address, void *memory_basic_info, u64 memory_basic_info_size); 65 W32(b32) UnmapViewOfFile(void *); 66 67 function b32 68 os_reserve_region_locks(void) 69 { 70 u8 buffer[1024]; 71 Stream sb = {.data = buffer, .cap = countof(buffer)}; 72 stream_append_str8(&sb, str8(OS_SHARED_MEMORY_NAME "_lock_")); 73 74 i32 start_index = sb.widx; 75 u32 reserved_count = 0; 76 for EachElement(os_w32_shared_memory_semaphores, it) { 77 stream_reset(&sb, start_index); 78 stream_append_u64(&sb, it); 79 stream_append_byte(&sb, 0); 80 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 81 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 82 break; 83 reserved_count++; 84 } 85 86 b32 result = reserved_count == countof(os_w32_shared_memory_semaphores); 87 if (!result) { 88 for (u32 i = 0; i < reserved_count; i++) 89 CloseHandle(os_w32_shared_memory_semaphores[i].value[0]); 90 } 91 92 return result; 93 } 94 95 function str8 96 os_open_shared_memory_area(char *name) 97 { 98 struct alignas(16) { 99 void *BaseAddress; 100 void *AllocationBase; 101 u32 AllocationProtect; 102 u32 __alignment1; 103 u64 RegionSize; 104 u32 State; 105 u32 Protect; 106 u32 Type; 107 u32 __alignment2; 108 } memory_basic_info; 109 110 str8 result = {0}; 111 iptr h = OpenFileMappingA(FILE_MAP_ALL_ACCESS, 0, name); 112 if (h != INVALID_FILE) { 113 // NOTE(rnp): a size of 0 maps the whole region, we can determine its size after 114 void *new = MapViewOfFile(h, FILE_MAP_ALL_ACCESS, 0, 0, 0); 115 if (new && 116 VirtualQuery(new, &memory_basic_info, sizeof(memory_basic_info)) == sizeof(memory_basic_info) && 117 os_reserve_region_locks()) 118 { 119 result.data = new; 120 result.length = (i64)memory_basic_info.RegionSize; 121 } 122 123 if (new && !result.data) 124 UnmapViewOfFile(new); 125 126 CloseHandle(h); 127 } 128 return result; 129 } 130 131 function void 132 os_close_shared_memory_area(void *memory, i64 size) 133 { 134 UnmapViewOfFile(memory); 135 } 136 137 #endif 138 139 #define lib_error_check(c, e) lib_error_check_(c, BeamformerLibErrorKind_##e) 140 function b32 141 lib_error_check_(b32 condition, BeamformerLibErrorKind error_kind) 142 { 143 b32 result = condition; 144 if (!result) g_beamformer_library_context.last_error = error_kind; 145 assert(result); 146 return result; 147 } 148 149 function b32 150 check_shared_memory(void) 151 { 152 b32 result = g_beamformer_library_context.bp != 0; 153 if unlikely(!g_beamformer_library_context.bp) { 154 str8 shared_memory = os_open_shared_memory_area(OS_SHARED_MEMORY_NAME); 155 if (lib_error_check(shared_memory.data != 0, SharedMemory)) { 156 BeamformerSharedMemory *bp = (BeamformerSharedMemory *)shared_memory.data; 157 result = lib_error_check(bp->version == BEAMFORMER_SHARED_MEMORY_VERSION, VersionMismatch); 158 if (result) { 159 g_beamformer_library_context.bp = bp; 160 g_beamformer_library_context.shared_memory_size = shared_memory.length; 161 } else { 162 os_close_shared_memory_area(shared_memory.data, shared_memory.length); 163 } 164 } 165 } 166 167 if likely(g_beamformer_library_context.bp) 168 result = lib_error_check(likely(!g_beamformer_library_context.bp->invalid), InvalidAccess); 169 return result; 170 } 171 172 function b32 173 valid_parameter_block(u32 block) 174 { 175 b32 result = check_shared_memory(); 176 if (result) { 177 result = lib_error_check(block < g_beamformer_library_context.bp->reserved_parameter_blocks, 178 ParameterBlockUnallocated); 179 } 180 return result; 181 } 182 183 function BeamformWork * 184 try_push_work_queue(void) 185 { 186 BeamformWork *result = beamform_work_queue_push(&g_beamformer_library_context.bp->external_work_queue); 187 lib_error_check(result != 0, WorkQueueFull); 188 return result; 189 } 190 191 function b32 192 lib_try_lock(i32 lock, i32 timeout_ms) 193 { 194 b32 result = beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, (u32)timeout_ms); 195 lib_error_check(result, SyncVariable); 196 return result; 197 } 198 199 function void 200 lib_release_lock(i32 lock) 201 { 202 beamformer_shared_memory_release_lock(g_beamformer_library_context.bp, lock); 203 } 204 205 u32 206 beamformer_get_api_version(void) 207 { 208 return BEAMFORMER_SHARED_MEMORY_VERSION; 209 } 210 211 const char * 212 beamformer_error_string(BeamformerLibErrorKind kind) 213 { 214 #define X(type, num, string) string, 215 local_persist const char *error_string_table[] = {BEAMFORMER_LIB_ERRORS "invalid error kind"}; 216 #undef X 217 return error_string_table[Min(kind, countof(error_string_table) - 1)]; 218 } 219 220 BeamformerLibErrorKind 221 beamformer_get_last_error(void) 222 { 223 return g_beamformer_library_context.last_error; 224 } 225 226 const char * 227 beamformer_get_last_error_string(void) 228 { 229 return beamformer_error_string(beamformer_get_last_error()); 230 } 231 232 void 233 beamformer_set_global_timeout(u32 timeout_ms) 234 { 235 g_beamformer_library_context.timeout_ms = timeout_ms; 236 } 237 238 b32 239 beamformer_reserve_parameter_blocks(uint32_t count) 240 { 241 b32 result = 0; 242 if (check_shared_memory() && 243 lib_error_check(count <= BeamformerMaxParameterBlocks, ParameterBlockOverflow)) 244 { 245 g_beamformer_library_context.bp->reserved_parameter_blocks = count; 246 result = 1; 247 } 248 return result; 249 } 250 251 function b32 252 validate_parameters(BeamformerParameters *bp) 253 { 254 if (!lib_error_check(Between(bp->contrast_mode, 0, BeamformerContrastMode_Count - 1), InvalidContrastMode)) 255 return 0; 256 257 u32 contrast_raw_sample_count = bp->acquisition_count * bp->sample_count * beamformer_contrast_mode_samples[bp->contrast_mode]; 258 if (!lib_error_check(contrast_raw_sample_count <= bp->raw_data_dimensions.x, DataSizeMismatch)) 259 return 0; 260 261 // NOTE(rnp): frame size checks 262 { 263 // TODO(rnp): this check is overly conservative, what if we are exporting something smaller than Float32Complex 264 u64 buffer_size = g_beamformer_library_context.bp->beamformed_frame_buffer_size; 265 u64 frame_size = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z) 266 * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex]; 267 u64 incoherent_size = frame_size / 2; 268 if (bp->coherency_weighting) 269 buffer_size -= incoherent_size; 270 271 if (!lib_error_check(frame_size <= buffer_size, FrameSizeOverflow)) 272 return 0; 273 } 274 275 return 1; 276 } 277 278 function b32 279 validate_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind) 280 { 281 b32 data_kind_test = Between(data_kind, 0, BeamformerDataKind_Count - 1); 282 if (!lib_error_check(data_kind_test, InvalidDataKind)) 283 return 0; 284 285 if (!lib_error_check(shader_count <= BeamformerMaxComputeShaderStages, ComputeStageOverflow)) 286 return 0; 287 288 for (u32 i = 0; i < shader_count; i++) { 289 b32 stage_test = Between(shaders[i], BeamformerShaderKind_ComputeFirst, BeamformerShaderKind_ComputeLast); 290 if (!lib_error_check(stage_test, InvalidComputeStage)) 291 return 0; 292 293 if (shaders[i] == BeamformerShaderKind_Hilbert && 294 !lib_error_check(g_beamformer_library_context.bp->capabilities.hilbert != 0, InvalidComputeStage)) 295 return 0; 296 297 if (shaders[i] == BeamformerShaderKind_Demodulate && 298 !lib_error_check(!beamformer_data_kind_complex[data_kind], InvalidDemodulationDataKind)) 299 { 300 return 0; 301 } 302 } 303 304 b32 start_stage_test = shaders[0] == BeamformerShaderKind_Demodulate || 305 shaders[0] == BeamformerShaderKind_Decode; 306 if (!lib_error_check(start_stage_test, InvalidStartShader)) 307 return 0; 308 309 return 1; 310 } 311 312 u64 313 beamformer_maximum_rf_data_size(void) 314 { 315 u64 result = U64_MAX; 316 if (check_shared_memory()) { 317 Arena *sm = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 318 g_beamformer_library_context.shared_memory_size); 319 result = Min(sm->reserved - sm->position, g_beamformer_library_context.bp->capabilities.max_rf_data_size); 320 } 321 return result; 322 } 323 324 u64 325 beamformer_maximum_frames_for_parameters(BeamformerParameters *bp) 326 { 327 u64 result = U64_MAX; 328 if (check_shared_memory() && validate_parameters(bp)) { 329 // TODO(rnp): overly conservative frame size check 330 u64 buffer_size = g_beamformer_library_context.bp->beamformed_frame_buffer_size; 331 u64 frame_size = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z) 332 * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex]; 333 u64 incoherent_size = frame_size / 2; 334 if (bp->coherency_weighting) 335 buffer_size -= incoherent_size; 336 result = buffer_size / frame_size; 337 } 338 return result; 339 } 340 341 u64 342 beamformer_maximum_frames_for_simple_parameters(BeamformerSimpleParameters *bp) 343 { 344 u64 result = beamformer_maximum_frames_for_parameters((BeamformerParameters *)bp); 345 return result; 346 } 347 348 function b32 349 parameter_block_region_upload(const void *data, u32 size, u32 block, u32 block_offset, i32 timeout_ms) 350 { 351 i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block; 352 b32 result = valid_parameter_block(block) && lib_try_lock(lock, timeout_ms); 353 if (result) { 354 memory_copy((u8 *)beamformer_parameter_block(g_beamformer_library_context.bp, block) + block_offset, 355 data, size); 356 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters); 357 lib_release_lock(lock); 358 } 359 return result; 360 } 361 362 b32 363 beamformer_set_pipeline_stage_parameters_at(u32 stage_index, i32 parameter, u32 block) 364 { 365 u32 offset = BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_ComputePipeline]; 366 offset += offsetof(BeamformerComputePipeline, parameters); 367 offset += (stage_index % BeamformerMaxComputeShaderStages) * sizeof(BeamformerShaderParameters); 368 b32 result = parameter_block_region_upload(¶meter, sizeof(BeamformerShaderParameters), block, 369 offset, g_beamformer_library_context.timeout_ms); 370 return result; 371 } 372 373 b32 374 beamformer_set_pipeline_stage_parameters(u32 stage_index, i32 parameter) 375 { 376 b32 result = beamformer_set_pipeline_stage_parameters_at(stage_index, parameter, 0); 377 return result; 378 } 379 380 b32 381 beamformer_push_pipeline_at(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind, u32 block) 382 { 383 b32 result = 0; 384 if (check_shared_memory() && validate_pipeline(shaders, shader_count, data_kind)) { 385 i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block; 386 if (valid_parameter_block(block) && lib_try_lock(lock, g_beamformer_library_context.timeout_ms)) { 387 BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block); 388 memory_copy(&b->pipeline.shaders, shaders, shader_count * sizeof(*shaders)); 389 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters); 390 b->pipeline.shader_count = shader_count; 391 b->pipeline.data_kind = data_kind; 392 lib_release_lock(lock); 393 result = 1; 394 } 395 } 396 return result; 397 } 398 399 b32 400 beamformer_push_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind) 401 { 402 b32 result = beamformer_push_pipeline_at(shaders, shader_count, data_kind, 0); 403 return result; 404 } 405 406 b32 407 beamformer_create_filter(BeamformerFilterParameters *filter, u8 filter_slot, u8 parameter_block) 408 { 409 b32 result = 0; 410 if (lib_error_check(filter->kind >= 0 && filter->kind < BeamformerFilterKind_Count, InvalidFilterKind)) { 411 if (check_shared_memory()) { 412 BeamformWork *work = try_push_work_queue(); 413 if (work) { 414 BeamformerCreateFilterContext *ctx = &work->create_filter_context; 415 work->kind = BeamformerWorkKind_CreateFilter; 416 ctx->parameters = *filter; 417 ctx->filter_slot = filter_slot % BeamformerFilterSlots; 418 ctx->parameter_block = parameter_block % BeamformerMaxParameterBlocks; 419 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 420 result = 1; 421 } 422 } 423 } 424 return result; 425 } 426 427 function void 428 beamformer_flush_commands(void) 429 { 430 i32 lock = BeamformerSharedMemoryLockKind_DispatchCompute; 431 beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, 0); 432 } 433 434 #define BEAMFORMER_UPLOAD_FNS \ 435 X(channel_mapping, i16, 1, ChannelMapping) \ 436 X(focal_vectors, f32, 2, FocalVectors) \ 437 X(sparse_elements, i16, 1, SparseElements) \ 438 X(transmit_receive_orientations, u8, 1, TransmitReceiveOrientations) 439 440 #define X(name, dtype, elements, region_name) \ 441 b32 beamformer_push_##name ##_at(const dtype *data, u32 count, u32 block) { \ 442 b32 result = 0; \ 443 if (lib_error_check(count <= countof(((BeamformerParameterBlock *)0)->name), BufferOverflow)) { \ 444 result = parameter_block_region_upload(data, count * elements * sizeof(dtype), block, \ 445 offsetof(BeamformerParameterBlock, name), \ 446 g_beamformer_library_context.timeout_ms); \ 447 } \ 448 return result; \ 449 } 450 BEAMFORMER_UPLOAD_FNS 451 #undef X 452 453 #define X(name, dtype, ...) \ 454 b32 beamformer_push_##name (const dtype *data, u32 count) { \ 455 b32 result = beamformer_push_##name ##_at(data, count, 0); \ 456 return result; \ 457 } 458 BEAMFORMER_UPLOAD_FNS 459 #undef X 460 461 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(name) void name(void *restrict output_v, \ 462 void *restrict input_v, \ 463 u32 sample_count) 464 typedef BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_fn); 465 466 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST \ 467 X(i16) \ 468 X(f32) \ 469 X(f16) \ 470 471 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), ""); 472 473 #define X(type, ...) \ 474 function BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_##type) \ 475 { \ 476 type *input_a = (type *)input_v + 0 * sample_count; \ 477 type *input_b = (type *)input_v + 1 * sample_count; \ 478 type *input_c = (type *)input_v + 2 * sample_count; \ 479 type *output = (type *)output_v; \ 480 for (u32 sample = 0; sample < sample_count; sample++) \ 481 output[sample] = input_a[sample] - input_b[sample] - input_c[sample]; \ 482 } 483 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST 484 #undef X 485 486 function b32 487 beamformer_push_data_base(void *data, u64 data_size, i32 timeout_ms, u32 block) 488 { 489 b32 result = 0; 490 Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 491 g_beamformer_library_context.shared_memory_size); 492 BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block); 493 BeamformerParameters *bp = &b->parameters; 494 BeamformerDataKind data_kind = b->pipeline.data_kind; 495 BeamformerContrastMode contrast_mode = bp->contrast_mode; 496 497 498 u64 arena_size = scratch->reserved - scratch->position; 499 u64 max_rf_size = g_beamformer_library_context.bp->capabilities.max_rf_data_size; 500 u32 rf_size = bp->acquisition_count * bp->sample_count * bp->channel_count * beamformer_data_kind_byte_size[data_kind]; 501 u32 raw_size = bp->raw_data_dimensions.x * bp->raw_data_dimensions.y * beamformer_data_kind_byte_size[data_kind]; 502 503 // TODO(rnp): support multi push upload so that max_rf_size is actual limit 504 if (lib_error_check(rf_size <= arena_size, BufferOverflow) && 505 lib_error_check(rf_size <= max_rf_size, RFDataSizeOverflow) && 506 lib_error_check(rf_size <= data_size && data_size == raw_size, DataSizeMismatch)) 507 { 508 if (lib_try_lock(BeamformerSharedMemoryLockKind_UploadRF, timeout_ms)) { 509 if (lib_try_lock(BeamformerSharedMemoryLockKind_ScratchSpace, 0)) { 510 u32 channel_count = bp->channel_count; 511 u32 out_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->sample_count * bp->acquisition_count; 512 u32 in_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->raw_data_dimensions.x; 513 514 for (u32 channel = 0; channel < channel_count; channel++) { 515 u16 data_channel = (u16)b->channel_mapping[channel]; 516 u32 out_off = out_channel_stride * channel; 517 u32 in_off = in_channel_stride * data_channel; 518 u8 *memory = (u8 *)scratch + scratch->position + out_off; 519 switch (contrast_mode) { 520 default:{ 521 /* NOTE(rnp): non temporal copy would be better, but we can't ensure 522 * 64 byte boundaries. */ 523 memory_copy(memory, (u8 *)data + in_off, out_channel_stride); 524 }break; 525 526 case BeamformerContrastMode_A1S2:{ 527 read_only u8 reduce_a1s2_index_map[] = { 528 [BeamformerDataKind_Int16] = 0, 529 [BeamformerDataKind_Int16Complex] = 0, 530 [BeamformerDataKind_Float32] = 1, 531 [BeamformerDataKind_Float32Complex] = 1, 532 [BeamformerDataKind_Float16] = 2, 533 [BeamformerDataKind_Float16Complex] = 2, 534 }; 535 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), ""); 536 537 local_persist beamformer_reduce_a1s2_contrast_fn *reduce_a1s2_fn_table[] = { 538 #define X(type, ...) beamformer_reduce_a1s2_contrast_##type, 539 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST 540 #undef X 541 }; 542 543 u32 sample_count = bp->sample_count * beamformer_data_kind_element_count[data_kind]; 544 u32 acquisition_stride = sample_count * beamformer_data_kind_element_size[data_kind]; 545 for (u32 acquisition = 0; acquisition < bp->acquisition_count; acquisition++) { 546 void *out = memory + acquisition * acquisition_stride; 547 void *in = (u8 *)data + in_off + 3 * acquisition * acquisition_stride; 548 reduce_a1s2_fn_table[reduce_a1s2_index_map[data_kind]](out, in, sample_count); 549 } 550 }break; 551 } 552 } 553 554 lib_release_lock(BeamformerSharedMemoryLockKind_ScratchSpace); 555 /* TODO(rnp): need a better way to communicate this */ 556 u64 rf_block_rf_size = ((u64)block << 56ull) | (rf_size & 0x00FFFFFFFFFFFFFFull); 557 atomic_store_u64(&g_beamformer_library_context.bp->rf_block_rf_size, rf_block_rf_size); 558 atomic_store_u64(&g_beamformer_library_context.bp->new_rf_upload, 1ull); 559 result = 1; 560 } 561 } 562 } 563 return result; 564 } 565 566 function b32 567 beamformer_start_compute(u64 offset, u32 image_plane_tag, u32 parameter_slot) 568 { 569 b32 result = 0; 570 if (check_shared_memory()) { 571 u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks; 572 if (lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) && 573 lib_error_check(image_plane_tag < BeamformerViewPlaneTag_Count, InvalidImagePlane)) 574 { 575 BeamformWork *work = try_push_work_queue(); 576 if (work) { 577 b64 new_rf = atomic_swap_u64(&g_beamformer_library_context.bp->new_rf_upload, 0ull); 578 work->kind = new_rf ? BeamformerWorkKind_WaitThenCompute : BeamformerWorkKind_Compute; 579 work->compute_context.view_plane = image_plane_tag; 580 work->compute_context.parameter_block = parameter_slot; 581 work->compute_context.rf_offset = offset; 582 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 583 beamformer_flush_commands(); 584 result = 1; 585 } 586 } 587 } 588 return result; 589 } 590 591 function b32 592 beamformer_push_data(void *data, u64 data_size, u32 parameter_slot) 593 { 594 b32 result = 0; 595 if (check_shared_memory()) { 596 u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks; 597 result = lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) && 598 beamformer_push_data_base(data, data_size, g_beamformer_library_context.timeout_ms, parameter_slot); 599 } 600 return result; 601 } 602 603 b32 604 beamformer_push_data_with_compute(void *data, u32 data_size, u32 image_plane_tag, u32 parameter_slot) 605 { 606 b32 result = beamformer_push_data(data, data_size, parameter_slot) && 607 beamformer_start_compute(0, image_plane_tag, parameter_slot); 608 return result; 609 } 610 611 b32 612 beamformer_push_parameters_at(BeamformerParameters *bp, u32 block) 613 { 614 b32 result = check_shared_memory() && validate_parameters(bp); 615 if (result) { 616 result = parameter_block_region_upload(bp, sizeof(*bp), block, 617 offsetof(BeamformerParameterBlock, parameters), 618 g_beamformer_library_context.timeout_ms); 619 if (result) { 620 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, 621 BeamformerParameterDirtyFlag_NotifyUI); 622 } 623 } 624 return result; 625 } 626 627 b32 628 beamformer_push_parameters(BeamformerParameters *bp) 629 { 630 b32 result = beamformer_push_parameters_at(bp, 0); 631 return result; 632 } 633 634 b32 635 beamformer_push_simple_parameters_at(BeamformerSimpleParameters *bp, u32 block) 636 { 637 b32 result = check_shared_memory(); 638 if (result) { 639 alignas(64) v2 focal_vectors[countof(bp->steering_angles)]; 640 for (u32 i = 0; i < countof(bp->steering_angles); i++) 641 focal_vectors[i] = (v2){{bp->steering_angles[i], bp->focal_depths[i]}}; 642 643 result &= beamformer_push_parameters_at((BeamformerParameters *)bp, block); 644 result &= beamformer_push_pipeline_at(bp->compute_stages, bp->compute_stages_count, (BeamformerDataKind)bp->data_kind, block); 645 result &= beamformer_push_channel_mapping_at(bp->channel_mapping, bp->channel_count, block); 646 result &= beamformer_push_focal_vectors_at((f32 *)focal_vectors, countof(focal_vectors), block); 647 result &= beamformer_push_transmit_receive_orientations_at(bp->transmit_receive_orientations, 648 bp->acquisition_count, block); 649 650 if (bp->acquisition_kind == BeamformerAcquisitionKind_UFORCES || 651 bp->acquisition_kind == BeamformerAcquisitionKind_UHERCULES) 652 { 653 result &= beamformer_push_sparse_elements_at(bp->sparse_elements, bp->acquisition_count, block); 654 } 655 656 for (u32 stage = 0; stage < bp->compute_stages_count; stage++) 657 result &= beamformer_set_pipeline_stage_parameters_at(stage, bp->compute_stage_parameters[stage], block); 658 } 659 return result; 660 } 661 662 b32 663 beamformer_push_simple_parameters(BeamformerSimpleParameters *bp) 664 { 665 b32 result = beamformer_push_simple_parameters_at(bp, 0); 666 return result; 667 } 668 669 function b32 670 beamformer_export_buffer(BeamformerExportContext export_context) 671 { 672 BeamformWork *work = try_push_work_queue(); 673 b32 result = work && lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, 0); 674 if (result) { 675 work->export_context = export_context; 676 work->kind = BeamformerWorkKind_ExportBuffer; 677 work->lock = BeamformerSharedMemoryLockKind_ScratchSpace; 678 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 679 } 680 return result; 681 } 682 683 function b32 684 beamformer_export(BeamformerExportContext export, void *out, i32 timeout_ms) 685 { 686 b32 result = 0; 687 if (beamformer_export_buffer(export)) { 688 /* NOTE(rnp): if this fails it just means that the work from push_data hasn't 689 * started yet. This is here to catch the other case where the work started 690 * and finished before we finished queuing the export work item */ 691 beamformer_flush_commands(); 692 693 if (lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, timeout_ms)) { 694 if (lib_try_lock(BeamformerSharedMemoryLockKind_ScratchSpace, 0)) { 695 void *sm = beamformer_shared_memory_data_pointer(g_beamformer_library_context.bp, 696 g_beamformer_library_context.shared_memory_size); 697 memory_copy(out, sm, export.size); 698 lib_release_lock(BeamformerSharedMemoryLockKind_ScratchSpace); 699 result = 1; 700 } 701 lib_release_lock(BeamformerSharedMemoryLockKind_ExportSync); 702 } 703 } 704 return result; 705 } 706 707 BEAMFORMER_LIB_EXPORT b32 708 beamformer_get_last_frames(void *out_data, u64 out_data_size, u32 count) 709 { 710 BeamformerExportContext export = {0}; 711 export.kind = BeamformerExportKind_BeamformedData; 712 export.count = count; 713 export.size = out_data_size; 714 b32 result = out_data && out_data_size && count && beamformer_export(export, out_data, g_beamformer_library_context.timeout_ms); 715 return result; 716 } 717 718 b32 719 beamformer_beamform_data(BeamformerSimpleParameters *bp, void *data, uint32_t data_size, 720 void *out_data, int32_t timeout_ms) 721 { 722 b32 result = beamformer_push_simple_parameters(bp); 723 if (result) { 724 beamformer_set_global_timeout(timeout_ms); 725 726 iv3 output_points = bp->output_points.xyz; 727 output_points.E[0] = Max(1, output_points.E[0]); 728 output_points.E[1] = Max(1, output_points.E[1]); 729 output_points.E[2] = Max(1, output_points.E[2]); 730 731 b32 complex = 0; 732 for (u32 stage = 0; stage < bp->compute_stages_count; stage++) { 733 BeamformerShaderKind shader = (BeamformerShaderKind)bp->compute_stages[stage]; 734 complex |= shader == BeamformerShaderKind_Demodulate || shader == BeamformerShaderKind_Hilbert; 735 } 736 737 u64 output_size = output_points.x * output_points.y * output_points.z * sizeof(f32); 738 if (complex) output_size *= 2; 739 740 Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 741 g_beamformer_library_context.shared_memory_size); 742 u64 scratch_size = scratch->reserved - scratch->position; 743 if (result && out_data) result &= lib_error_check(output_size <= scratch_size, ExportSpaceOverflow); 744 745 if (result) { 746 result = beamformer_push_data_with_compute(data, data_size, 0, 0); 747 if (result && out_data) 748 result = beamformer_get_last_frames(out_data, output_size, 1); 749 } 750 } 751 return result; 752 } 753 754 function b32 755 beamformer_compute_timings(BeamformerComputeStatsTable *output, i32 timeout_ms) 756 { 757 b32 result = 0; 758 if (check_shared_memory()) { 759 Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp, 760 g_beamformer_library_context.shared_memory_size); 761 u64 scratch_size = scratch->reserved - scratch->position; 762 if (lib_error_check(sizeof(*output) <= scratch_size, ExportSpaceOverflow)) { 763 BeamformerExportContext export = {0}; 764 export.kind = BeamformerExportKind_Stats; 765 export.size = sizeof(*output); 766 result = beamformer_export(export, output, timeout_ms); 767 } 768 } 769 return result; 770 } 771 772 i32 773 beamformer_live_parameters_get_dirty_flag(void) 774 { 775 i32 result = -1; 776 if (check_shared_memory()) { 777 u32 flag = ctz_u64(g_beamformer_library_context.bp->live_imaging_dirty_flags); 778 if (flag != 64) { 779 atomic_and_u32(&g_beamformer_library_context.bp->live_imaging_dirty_flags, ~(1u << flag)); 780 result = (i32)flag; 781 } 782 } 783 return result; 784 } 785 786 BeamformerLiveImagingParameters * 787 beamformer_get_live_parameters(void) 788 { 789 BeamformerLiveImagingParameters *result = 0; 790 if (check_shared_memory()) result = &g_beamformer_library_context.bp->live_imaging_parameters; 791 return result; 792 } 793 794 b32 795 beamformer_set_live_parameters(BeamformerLiveImagingParameters *new) 796 { 797 b32 result = 0; 798 if (check_shared_memory()) { 799 memory_copy(&g_beamformer_library_context.bp->live_imaging_parameters, new, sizeof(*new)); 800 store_fence(); 801 result = 1; 802 } 803 return result; 804 }