ogl_beamformer_lib.c (27884B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_IMPORT static 3 4 #ifndef BASE_PLATFORM_H 5 #define BASE_PLATFORM_NO_MAIN 1 6 #define BASE_EXPORT static 7 #endif 8 9 #include "../util.h" 10 11 #include "../generated/beamformer.c" 12 #include "ogl_beamformer_lib_base.h" 13 14 #if OS_LINUX 15 #include "../base_linux.c" 16 #elif OS_WINDOWS 17 #include "../base_win32.c" 18 19 W32(iptr) OpenFileMappingA(u32, b32, c8 *); 20 21 #else 22 #error Unsupported Platform 23 #endif 24 25 #include "../beamformer_compute_stats.c" 26 #include "../beamformer_shared_memory.c" 27 28 global struct { 29 BeamformerSharedMemory *bp; 30 b32 new_rf; 31 i32 timeout_ms; 32 BeamformerLibErrorKind last_error; 33 } g_beamformer_library_context; 34 35 #if OS_LINUX 36 37 function str8 38 os_open_shared_memory_area(char *name) 39 { 40 str8 result = {0}; 41 i32 fd = shm_open(name, O_RDWR, S_IRUSR|S_IWUSR); 42 if (fd > 0) { 43 struct stat sb; 44 if (fstat(fd, &sb) != -1) { 45 void *new = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); 46 if (new != MAP_FAILED) { 47 result.data = new; 48 result.length = sb.st_size; 49 } 50 } 51 close(fd); 52 } 53 return result; 54 } 55 56 function void 57 os_close_shared_memory_area(void *memory, i64 size) 58 { 59 munmap(memory, size); 60 } 61 62 #elif OS_WINDOWS 63 64 W32(u64) VirtualQuery(void *base_address, void *memory_basic_info, u64 memory_basic_info_size); 65 W32(b32) UnmapViewOfFile(void *); 66 67 function b32 68 os_reserve_region_locks(void) 69 { 70 u8 buffer[1024]; 71 Stream sb = {.data = buffer, .cap = countof(buffer)}; 72 stream_append_str8(&sb, str8(OS_SHARED_MEMORY_NAME "_lock_")); 73 74 i32 start_index = sb.widx; 75 u32 reserved_count = 0; 76 for EachElement(os_w32_shared_memory_semaphores, it) { 77 stream_reset(&sb, start_index); 78 stream_append_u64(&sb, it); 79 stream_append_byte(&sb, 0); 80 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 81 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 82 break; 83 reserved_count++; 84 } 85 86 b32 result = reserved_count == countof(os_w32_shared_memory_semaphores); 87 if (!result) { 88 for (u32 i = 0; i < reserved_count; i++) 89 CloseHandle(os_w32_shared_memory_semaphores[i].value[0]); 90 } 91 92 return result; 93 } 94 95 function str8 96 os_open_shared_memory_area(char *name) 97 { 98 struct alignas(16) { 99 void *BaseAddress; 100 void *AllocationBase; 101 u32 AllocationProtect; 102 u32 __alignment1; 103 u64 RegionSize; 104 u32 State; 105 u32 Protect; 106 u32 Type; 107 u32 __alignment2; 108 } memory_basic_info; 109 110 str8 result = {0}; 111 iptr h = OpenFileMappingA(FILE_MAP_ALL_ACCESS, 0, name); 112 if (h != INVALID_FILE) { 113 // NOTE(rnp): a size of 0 maps the whole region, we can determine its size after 114 void *new = MapViewOfFile(h, FILE_MAP_ALL_ACCESS, 0, 0, 0); 115 if (new && 116 VirtualQuery(new, &memory_basic_info, sizeof(memory_basic_info)) == sizeof(memory_basic_info) && 117 os_reserve_region_locks()) 118 { 119 result.data = new; 120 result.length = (i64)memory_basic_info.RegionSize; 121 } 122 123 if (new && !result.data) 124 UnmapViewOfFile(new); 125 126 CloseHandle(h); 127 } 128 return result; 129 } 130 131 function void 132 os_close_shared_memory_area(void *memory, i64 size) 133 { 134 UnmapViewOfFile(memory); 135 } 136 137 #endif 138 139 #define lib_error_check(c, e) lib_error_check_(c, BeamformerLibErrorKind_##e) 140 function b32 141 lib_error_check_(b32 condition, BeamformerLibErrorKind error_kind) 142 { 143 b32 result = condition; 144 if (!result) g_beamformer_library_context.last_error = error_kind; 145 assert(result); 146 return result; 147 } 148 149 function b32 150 check_shared_memory(void) 151 { 152 b32 result = g_beamformer_library_context.bp != 0; 153 if unlikely(!g_beamformer_library_context.bp) { 154 str8 shared_memory = os_open_shared_memory_area(OS_SHARED_MEMORY_NAME); 155 if (lib_error_check(shared_memory.data != 0, SharedMemory)) { 156 BeamformerSharedMemory *bp = (BeamformerSharedMemory *)shared_memory.data; 157 result = lib_error_check(bp->version == BEAMFORMER_SHARED_MEMORY_VERSION, VersionMismatch); 158 if (result) { 159 g_beamformer_library_context.bp = bp; 160 } else { 161 os_close_shared_memory_area(shared_memory.data, shared_memory.length); 162 } 163 } 164 } 165 166 if likely(g_beamformer_library_context.bp) 167 result = lib_error_check(likely(!g_beamformer_library_context.bp->invalid), InvalidAccess); 168 return result; 169 } 170 171 function b32 172 valid_parameter_block(u32 block) 173 { 174 b32 result = check_shared_memory(); 175 if (result) { 176 result = lib_error_check(block < g_beamformer_library_context.bp->reserved_parameter_blocks, 177 ParameterBlockUnallocated); 178 } 179 return result; 180 } 181 182 function BeamformWork * 183 try_push_work_queue(void) 184 { 185 BeamformWork *result = beamform_work_queue_push(&g_beamformer_library_context.bp->external_work_queue); 186 lib_error_check(result != 0, WorkQueueFull); 187 return result; 188 } 189 190 function b32 191 lib_try_lock(i32 lock, i32 timeout_ms) 192 { 193 b32 result = beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, (u32)timeout_ms); 194 lib_error_check(result, SyncVariable); 195 return result; 196 } 197 198 function void 199 lib_release_lock(i32 lock) 200 { 201 beamformer_shared_memory_release_lock(g_beamformer_library_context.bp, lock); 202 } 203 204 u32 205 beamformer_get_api_version(void) 206 { 207 return BEAMFORMER_SHARED_MEMORY_VERSION; 208 } 209 210 const char * 211 beamformer_error_string(BeamformerLibErrorKind kind) 212 { 213 #define X(type, num, string) string, 214 local_persist const char *error_string_table[] = {BEAMFORMER_LIB_ERRORS "invalid error kind"}; 215 #undef X 216 return error_string_table[Min(kind, countof(error_string_table) - 1)]; 217 } 218 219 BeamformerLibErrorKind 220 beamformer_get_last_error(void) 221 { 222 return g_beamformer_library_context.last_error; 223 } 224 225 const char * 226 beamformer_get_last_error_string(void) 227 { 228 return beamformer_error_string(beamformer_get_last_error()); 229 } 230 231 void 232 beamformer_set_global_timeout(u32 timeout_ms) 233 { 234 g_beamformer_library_context.timeout_ms = timeout_ms; 235 } 236 237 b32 238 beamformer_reserve_parameter_blocks(uint32_t count) 239 { 240 b32 result = 0; 241 if (check_shared_memory() && 242 lib_error_check(count <= BeamformerMaxParameterBlocks, ParameterBlockOverflow)) 243 { 244 g_beamformer_library_context.bp->reserved_parameter_blocks = count; 245 result = 1; 246 } 247 return result; 248 } 249 250 function b32 251 validate_parameters(BeamformerParameters *bp) 252 { 253 if (!lib_error_check(Between(bp->contrast_mode, 0, BeamformerContrastMode_Count - 1), InvalidContrastMode)) 254 return 0; 255 256 u32 contrast_raw_sample_count = bp->acquisition_count * bp->sample_count * beamformer_contrast_mode_samples[bp->contrast_mode]; 257 if (!lib_error_check(contrast_raw_sample_count <= bp->raw_data_dimensions.x, DataSizeMismatch)) 258 return 0; 259 260 // NOTE(rnp): frame size checks 261 { 262 // TODO(rnp): this check is overly conservative, what if we are exporting something smaller than Float32Complex 263 u64 buffer_size = g_beamformer_library_context.bp->beamformed_frame_buffer_size; 264 u64 frame_size = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z) 265 * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex]; 266 u64 incoherent_size = frame_size / 2; 267 if (bp->coherency_weighting) 268 buffer_size -= incoherent_size; 269 270 if (!lib_error_check(frame_size <= buffer_size, FrameSizeOverflow)) 271 return 0; 272 } 273 274 return 1; 275 } 276 277 function b32 278 validate_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind) 279 { 280 b32 data_kind_test = Between(data_kind, 0, BeamformerDataKind_Count - 1); 281 if (!lib_error_check(data_kind_test, InvalidDataKind)) 282 return 0; 283 284 if (!lib_error_check(shader_count <= BeamformerMaxComputeShaderStages, ComputeStageOverflow)) 285 return 0; 286 287 for (u32 i = 0; i < shader_count; i++) { 288 b32 stage_test = Between(shaders[i], BeamformerShaderKind_ComputeFirst, BeamformerShaderKind_ComputeLast); 289 if (!lib_error_check(stage_test, InvalidComputeStage)) 290 return 0; 291 292 if (shaders[i] == BeamformerShaderKind_Hilbert && 293 !lib_error_check(g_beamformer_library_context.bp->capabilities.hilbert != 0, InvalidComputeStage)) 294 return 0; 295 296 if (shaders[i] == BeamformerShaderKind_Demodulate && 297 !lib_error_check(!beamformer_data_kind_complex[data_kind], InvalidDemodulationDataKind)) 298 { 299 return 0; 300 } 301 } 302 303 b32 start_stage_test = shaders[0] == BeamformerShaderKind_Demodulate || 304 shaders[0] == BeamformerShaderKind_Decode; 305 if (!lib_error_check(start_stage_test, InvalidStartShader)) 306 return 0; 307 308 return 1; 309 } 310 311 u64 312 beamformer_maximum_rf_data_size(void) 313 { 314 u64 result = U64_MAX; 315 if (check_shared_memory()) { 316 u64 transient_size = g_beamformer_library_context.bp->transient.page_count 317 * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE; 318 result = Min(transient_size, g_beamformer_library_context.bp->capabilities.max_rf_data_size); 319 } 320 return result; 321 } 322 323 u64 324 beamformer_maximum_frames_for_parameters(BeamformerParameters *bp) 325 { 326 u64 result = U64_MAX; 327 if (check_shared_memory() && validate_parameters(bp)) { 328 // TODO(rnp): overly conservative frame size check 329 u64 buffer_size = g_beamformer_library_context.bp->beamformed_frame_buffer_size; 330 u64 frame_size = Max(1, bp->output_points.x) * Max(1, bp->output_points.y) * Max(1, bp->output_points.z) 331 * beamformer_data_kind_byte_size[BeamformerDataKind_Float32Complex]; 332 u64 incoherent_size = frame_size / 2; 333 if (bp->coherency_weighting) 334 buffer_size -= incoherent_size; 335 result = buffer_size / frame_size; 336 } 337 return result; 338 } 339 340 u64 341 beamformer_maximum_frames_for_simple_parameters(BeamformerSimpleParameters *bp) 342 { 343 u64 result = beamformer_maximum_frames_for_parameters((BeamformerParameters *)bp); 344 return result; 345 } 346 347 function b32 348 parameter_block_region_upload(const void *data, u32 size, u32 block, u32 block_offset, i32 timeout_ms) 349 { 350 i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block; 351 b32 result = valid_parameter_block(block) && lib_try_lock(lock, timeout_ms); 352 if (result) { 353 memory_copy((u8 *)beamformer_parameter_block(g_beamformer_library_context.bp, block) + block_offset, 354 data, size); 355 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters); 356 lib_release_lock(lock); 357 } 358 return result; 359 } 360 361 b32 362 beamformer_set_pipeline_stage_parameters_at(u32 stage_index, i32 parameter, u32 block) 363 { 364 u32 offset = BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_ComputePipeline]; 365 offset += offsetof(BeamformerComputePipeline, parameters); 366 offset += (stage_index % BeamformerMaxComputeShaderStages) * sizeof(BeamformerShaderParameters); 367 b32 result = parameter_block_region_upload(¶meter, sizeof(BeamformerShaderParameters), block, 368 offset, g_beamformer_library_context.timeout_ms); 369 return result; 370 } 371 372 b32 373 beamformer_set_pipeline_stage_parameters(u32 stage_index, i32 parameter) 374 { 375 b32 result = beamformer_set_pipeline_stage_parameters_at(stage_index, parameter, 0); 376 return result; 377 } 378 379 b32 380 beamformer_push_pipeline_at(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind, u32 block) 381 { 382 b32 result = 0; 383 if (check_shared_memory() && validate_pipeline(shaders, shader_count, data_kind)) { 384 i32 lock = BeamformerSharedMemoryLockKind_Count + (i32)block; 385 if (valid_parameter_block(block) && lib_try_lock(lock, g_beamformer_library_context.timeout_ms)) { 386 BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block); 387 memory_copy(&b->pipeline.shaders, shaders, shader_count * sizeof(*shaders)); 388 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, BeamformerParameterDirtyFlag_Parameters); 389 b->pipeline.shader_count = shader_count; 390 b->pipeline.data_kind = data_kind; 391 lib_release_lock(lock); 392 result = 1; 393 } 394 } 395 return result; 396 } 397 398 b32 399 beamformer_push_pipeline(i32 *shaders, u32 shader_count, BeamformerDataKind data_kind) 400 { 401 b32 result = beamformer_push_pipeline_at(shaders, shader_count, data_kind, 0); 402 return result; 403 } 404 405 b32 406 beamformer_create_filter(BeamformerFilterParameters *filter, u8 filter_slot, u8 parameter_block) 407 { 408 b32 result = 0; 409 if (lib_error_check(filter->kind >= 0 && filter->kind < BeamformerFilterKind_Count, InvalidFilterKind)) { 410 if (check_shared_memory()) { 411 BeamformWork *work = try_push_work_queue(); 412 if (work) { 413 BeamformerCreateFilterContext *ctx = &work->create_filter_context; 414 work->kind = BeamformerWorkKind_CreateFilter; 415 ctx->parameters = *filter; 416 ctx->filter_slot = filter_slot % BeamformerFilterSlots; 417 ctx->parameter_block = parameter_block % BeamformerMaxParameterBlocks; 418 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 419 result = 1; 420 } 421 } 422 } 423 return result; 424 } 425 426 function void 427 beamformer_flush_commands(void) 428 { 429 i32 lock = BeamformerSharedMemoryLockKind_DispatchCompute; 430 beamformer_shared_memory_take_lock(g_beamformer_library_context.bp, lock, 0); 431 } 432 433 #define BEAMFORMER_UPLOAD_FNS \ 434 X(channel_mapping, i16, 1, ChannelMapping) \ 435 X(focal_vectors, f32, 2, FocalVectors) \ 436 X(sparse_elements, i16, 1, SparseElements) \ 437 X(transmit_receive_orientations, u8, 1, TransmitReceiveOrientations) 438 439 #define X(name, dtype, elements, region_name) \ 440 b32 beamformer_push_##name ##_at(const dtype *data, u32 count, u32 block) { \ 441 b32 result = 0; \ 442 if (lib_error_check(count <= countof(((BeamformerParameterBlock *)0)->name), BufferOverflow)) { \ 443 result = parameter_block_region_upload(data, count * elements * sizeof(dtype), block, \ 444 offsetof(BeamformerParameterBlock, name), \ 445 g_beamformer_library_context.timeout_ms); \ 446 } \ 447 return result; \ 448 } 449 BEAMFORMER_UPLOAD_FNS 450 #undef X 451 452 #define X(name, dtype, ...) \ 453 b32 beamformer_push_##name (const dtype *data, u32 count) { \ 454 b32 result = beamformer_push_##name ##_at(data, count, 0); \ 455 return result; \ 456 } 457 BEAMFORMER_UPLOAD_FNS 458 #undef X 459 460 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(name) void name(void *restrict output_v, \ 461 void *restrict input_v, \ 462 u32 sample_count) 463 typedef BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_fn); 464 465 #define BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST \ 466 X(i16) \ 467 X(f32) \ 468 X(f16) \ 469 470 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), ""); 471 472 #define X(type, ...) \ 473 function BEAMFORMER_REDUCE_A1S2_CONTRAST_FN(beamformer_reduce_a1s2_contrast_##type) \ 474 { \ 475 type *input_a = (type *)input_v + 0 * sample_count; \ 476 type *input_b = (type *)input_v + 1 * sample_count; \ 477 type *input_c = (type *)input_v + 2 * sample_count; \ 478 type *output = (type *)output_v; \ 479 for (u32 sample = 0; sample < sample_count; sample++) \ 480 output[sample] = input_a[sample] - input_b[sample] - input_c[sample]; \ 481 } 482 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST 483 #undef X 484 485 function b32 486 beamformer_push_data_base(void *data, u64 data_size, i32 timeout_ms, u32 block) 487 { 488 b32 result = 0; 489 BeamformerSharedMemory *sm = g_beamformer_library_context.bp; 490 BeamformerParameterBlock *b = beamformer_parameter_block(sm, block); 491 BeamformerParameters *bp = &b->parameters; 492 BeamformerDataKind data_kind = b->pipeline.data_kind; 493 BeamformerContrastMode contrast_mode = bp->contrast_mode; 494 495 u64 max_rf_size = sm->capabilities.max_rf_data_size; 496 u32 rf_size = bp->acquisition_count * bp->sample_count * bp->channel_count * beamformer_data_kind_byte_size[data_kind]; 497 u32 raw_size = bp->raw_data_dimensions.x * bp->raw_data_dimensions.y * beamformer_data_kind_byte_size[data_kind]; 498 499 // TODO(rnp): support multi push upload so that max_rf_size is actual limit 500 if (lib_try_lock(BeamformerSharedMemoryLockKind_UploadRF, timeout_ms)) { 501 BeamformerSharedMemoryAllocationHandle rf_handle = beamformer_shared_memory_transient_alloc(sm, .size = rf_size, .align = BEAMFORMER_RF_ALIGN); 502 if (lib_error_check(rf_handle != 0, TransientSpaceOverflow) && 503 lib_error_check(rf_size <= max_rf_size, RFDataSizeOverflow) && 504 lib_error_check(rf_size <= data_size && data_size == raw_size, DataSizeMismatch)) 505 { 506 u32 channel_count = bp->channel_count; 507 u32 out_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->sample_count * bp->acquisition_count; 508 u32 in_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->raw_data_dimensions.x; 509 510 u8 *base = beamformer_shared_memory_allocation_pointer(sm, rf_handle); 511 for (u32 channel = 0; channel < channel_count; channel++) { 512 u16 data_channel = (u16)b->channel_mapping[channel]; 513 u32 out_off = out_channel_stride * channel; 514 u32 in_off = in_channel_stride * data_channel; 515 516 u8 *memory = base + out_off; 517 switch (contrast_mode) { 518 default:{ 519 /* NOTE(rnp): non temporal copy would be better, but we can't ensure 520 * 64 byte boundaries. */ 521 memory_copy(memory, (u8 *)data + in_off, out_channel_stride); 522 }break; 523 524 case BeamformerContrastMode_A1S2:{ 525 read_only u8 reduce_a1s2_index_map[] = { 526 [BeamformerDataKind_Int16] = 0, 527 [BeamformerDataKind_Int16Complex] = 0, 528 [BeamformerDataKind_Float32] = 1, 529 [BeamformerDataKind_Float32Complex] = 1, 530 [BeamformerDataKind_Float16] = 2, 531 [BeamformerDataKind_Float16Complex] = 2, 532 }; 533 static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), ""); 534 535 local_persist beamformer_reduce_a1s2_contrast_fn *reduce_a1s2_fn_table[] = { 536 #define X(type, ...) beamformer_reduce_a1s2_contrast_##type, 537 BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST 538 #undef X 539 }; 540 beamformer_reduce_a1s2_contrast_fn *reduce = reduce_a1s2_fn_table[reduce_a1s2_index_map[data_kind]]; 541 542 u32 sample_count = bp->sample_count * beamformer_data_kind_element_count[data_kind]; 543 u32 acquisition_stride = sample_count * beamformer_data_kind_element_size[data_kind]; 544 for (u32 acquisition = 0; acquisition < bp->acquisition_count; acquisition++) { 545 void *out = memory + acquisition * acquisition_stride; 546 void *in = (u8 *)data + in_off + 3 * acquisition * acquisition_stride; 547 reduce(out, in, sample_count); 548 } 549 }break; 550 } 551 } 552 553 /* TODO(rnp): need a better way to communicate this */ 554 atomic_store_u64(&sm->rf_handle, rf_handle); 555 g_beamformer_library_context.new_rf = 1; 556 result = 1; 557 } 558 559 if (!result && rf_handle) { 560 beamformer_shared_memory_allocation_release(sm, rf_handle); 561 lib_release_lock(BeamformerSharedMemoryLockKind_UploadRF); 562 } 563 } 564 return result; 565 } 566 567 function b32 568 beamformer_start_compute(u64 offset, u32 image_plane_tag, u32 parameter_slot) 569 { 570 b32 result = 0; 571 if (check_shared_memory()) { 572 u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks; 573 if (lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) && 574 lib_error_check(image_plane_tag < BeamformerViewPlaneTag_Count, InvalidImagePlane)) 575 { 576 // TODO(rnp): this stuff will be passed in rather than taken from the existing set 577 BeamformerParameterBlock *pb = beamformer_parameter_block(g_beamformer_library_context.bp, parameter_slot); 578 BeamformerParameters *bp = &pb->parameters; 579 580 BeamformWork *work = try_push_work_queue(); 581 if (work) { 582 b32 new_rf = g_beamformer_library_context.new_rf; 583 g_beamformer_library_context.new_rf = 0; 584 work->kind = new_rf ? BeamformerWorkKind_WaitThenCompute : BeamformerWorkKind_Compute; 585 work->compute_context.view_plane = image_plane_tag; 586 work->compute_context.parameter_set = parameter_slot; 587 work->compute_context.rf_offset = offset; 588 work->compute_context.steering_angle = bp->single_focus ? bp->focal_vector.x : pb->focal_vectors[0].x; 589 work->compute_context.focus_depth = bp->single_focus ? bp->focal_vector.y : pb->focal_vectors[0].y; 590 work->compute_context.tx_rx_orientation = bp->single_orientation ? bp->transmit_receive_orientation 591 : pb->transmit_receive_orientations[0]; 592 memory_copy(&work->compute_context.voxel_transform, &bp->das_voxel_transform, sizeof(bp->das_voxel_transform)); 593 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 594 beamformer_flush_commands(); 595 result = 1; 596 } 597 } 598 } 599 return result; 600 } 601 602 function b32 603 beamformer_push_data(void *data, u64 data_size, u32 parameter_slot) 604 { 605 b32 result = 0; 606 if (check_shared_memory()) { 607 u32 reserved_blocks = g_beamformer_library_context.bp->reserved_parameter_blocks; 608 result = lib_error_check(parameter_slot < reserved_blocks, ParameterBlockUnallocated) && 609 beamformer_push_data_base(data, data_size, g_beamformer_library_context.timeout_ms, parameter_slot); 610 } 611 return result; 612 } 613 614 b32 615 beamformer_push_data_with_compute(void *data, u32 data_size, u32 image_plane_tag, u32 parameter_slot) 616 { 617 b32 result = beamformer_push_data(data, data_size, parameter_slot) && 618 beamformer_start_compute(0, image_plane_tag, parameter_slot); 619 return result; 620 } 621 622 b32 623 beamformer_push_parameters_at(BeamformerParameters *bp, u32 block) 624 { 625 b32 result = check_shared_memory() && validate_parameters(bp); 626 if (result) { 627 result = parameter_block_region_upload(bp, sizeof(*bp), block, 628 offsetof(BeamformerParameterBlock, parameters), 629 g_beamformer_library_context.timeout_ms); 630 if (result) { 631 beamformed_shared_memory_dirty_flag(g_beamformer_library_context.bp, block, 632 BeamformerParameterDirtyFlag_NotifyUI); 633 } 634 } 635 return result; 636 } 637 638 b32 639 beamformer_push_parameters(BeamformerParameters *bp) 640 { 641 b32 result = beamformer_push_parameters_at(bp, 0); 642 return result; 643 } 644 645 b32 646 beamformer_push_simple_parameters_at(BeamformerSimpleParameters *bp, u32 block) 647 { 648 b32 result = check_shared_memory(); 649 if (result) { 650 alignas(64) v2 focal_vectors[countof(bp->steering_angles)]; 651 for (u32 i = 0; i < countof(bp->steering_angles); i++) 652 focal_vectors[i] = (v2){{bp->steering_angles[i], bp->focal_depths[i]}}; 653 654 result &= beamformer_push_parameters_at((BeamformerParameters *)bp, block); 655 result &= beamformer_push_pipeline_at(bp->compute_stages, bp->compute_stages_count, (BeamformerDataKind)bp->data_kind, block); 656 result &= beamformer_push_channel_mapping_at(bp->channel_mapping, bp->channel_count, block); 657 result &= beamformer_push_focal_vectors_at((f32 *)focal_vectors, countof(focal_vectors), block); 658 result &= beamformer_push_transmit_receive_orientations_at(bp->transmit_receive_orientations, 659 bp->acquisition_count, block); 660 661 if (bp->acquisition_kind == BeamformerAcquisitionKind_UFORCES || 662 bp->acquisition_kind == BeamformerAcquisitionKind_UHERCULES) 663 { 664 result &= beamformer_push_sparse_elements_at(bp->sparse_elements, bp->acquisition_count, block); 665 } 666 667 for (u32 stage = 0; stage < bp->compute_stages_count; stage++) 668 result &= beamformer_set_pipeline_stage_parameters_at(stage, bp->compute_stage_parameters[stage], block); 669 } 670 return result; 671 } 672 673 b32 674 beamformer_push_simple_parameters(BeamformerSimpleParameters *bp) 675 { 676 b32 result = beamformer_push_simple_parameters_at(bp, 0); 677 return result; 678 } 679 680 function b32 681 beamformer_export_buffer(BeamformerExportContext export_context) 682 { 683 BeamformWork *work = try_push_work_queue(); 684 b32 result = work && lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, 0); 685 if (result) { 686 work->export_context = export_context; 687 work->kind = BeamformerWorkKind_ExportBuffer; 688 beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue); 689 } 690 return result; 691 } 692 693 function b32 694 beamformer_export(BeamformerExportContext export, void *out, i32 timeout_ms) 695 { 696 b32 result = 0; 697 if (beamformer_export_buffer(export)) { 698 /* NOTE(rnp): if this fails it just means that the work from push_data hasn't 699 * started yet. This is here to catch the other case where the work started 700 * and finished before we finished queuing the export work item */ 701 beamformer_flush_commands(); 702 703 if (lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, timeout_ms)) { 704 BeamformerSharedMemory *sm = g_beamformer_library_context.bp; 705 memory_copy(out, beamformer_shared_memory_allocation_pointer(sm, export.memory), 706 beamformer_shared_memory_allocation_size(sm, export.memory)); 707 lib_release_lock(BeamformerSharedMemoryLockKind_ExportSync); 708 result = 1; 709 } 710 } 711 return result; 712 } 713 714 BEAMFORMER_LIB_EXPORT b32 715 beamformer_get_last_frames(void *out_data, u64 out_data_size, u32 count) 716 { 717 BeamformerExportContext export = {0}; 718 export.kind = BeamformerExportKind_BeamformedData; 719 export.count = count; 720 if (out_data_size) { 721 export.memory = beamformer_shared_memory_transient_alloc(g_beamformer_library_context.bp, 722 .size = out_data_size, .align = 64); 723 } 724 725 b32 result = lib_error_check(export.memory != 0, TransientSpaceOverflow) && count; 726 if (result) 727 result = beamformer_export(export, out_data, g_beamformer_library_context.timeout_ms); 728 if (export.memory) 729 beamformer_shared_memory_allocation_release(g_beamformer_library_context.bp, export.memory); 730 731 return result; 732 } 733 734 b32 735 beamformer_beamform_data(BeamformerSimpleParameters *bp, void *data, uint32_t data_size, 736 void *out_data, int32_t timeout_ms) 737 { 738 b32 result = beamformer_push_simple_parameters(bp); 739 if (result) { 740 beamformer_set_global_timeout(timeout_ms); 741 742 iv3 output_points = bp->output_points.xyz; 743 output_points.E[0] = Max(1, output_points.E[0]); 744 output_points.E[1] = Max(1, output_points.E[1]); 745 output_points.E[2] = Max(1, output_points.E[2]); 746 747 b32 complex = 0; 748 for (u32 stage = 0; stage < bp->compute_stages_count; stage++) { 749 BeamformerShaderKind shader = (BeamformerShaderKind)bp->compute_stages[stage]; 750 complex |= shader == BeamformerShaderKind_Demodulate || shader == BeamformerShaderKind_Hilbert; 751 } 752 753 u64 output_size = output_points.x * output_points.y * output_points.z * sizeof(f32); 754 if (complex) output_size *= 2; 755 756 result = beamformer_push_data_with_compute(data, data_size, 0, 0); 757 758 if (result && out_data) 759 result = beamformer_get_last_frames(out_data, output_size, 1); 760 } 761 return result; 762 } 763 764 function b32 765 beamformer_compute_timings(BeamformerComputeStatsTable *output, i32 timeout_ms) 766 { 767 b32 result = 0; 768 if (check_shared_memory()) { 769 BeamformerExportContext export = {0}; 770 export.kind = BeamformerExportKind_Stats; 771 export.memory = beamformer_shared_memory_transient_alloc(g_beamformer_library_context.bp, 772 .size = sizeof(BeamformerComputeStatsTable), 773 .align = alignof(BeamformerComputeStatsTable)); 774 if (lib_error_check(export.memory != 0, TransientSpaceOverflow)) { 775 result = beamformer_export(export, output, timeout_ms); 776 beamformer_shared_memory_allocation_release(g_beamformer_library_context.bp, export.memory); 777 } 778 } 779 return result; 780 } 781 782 i32 783 beamformer_live_parameters_get_dirty_flag(void) 784 { 785 i32 result = -1; 786 if (check_shared_memory()) { 787 u32 flag = ctz_u64(g_beamformer_library_context.bp->live_imaging_dirty_flags); 788 if (flag != 64) { 789 atomic_and_u32(&g_beamformer_library_context.bp->live_imaging_dirty_flags, ~(1u << flag)); 790 result = (i32)flag; 791 } 792 } 793 return result; 794 } 795 796 BeamformerLiveImagingParameters * 797 beamformer_get_live_parameters(void) 798 { 799 BeamformerLiveImagingParameters *result = 0; 800 if (check_shared_memory()) result = &g_beamformer_library_context.bp->live_imaging_parameters; 801 return result; 802 } 803 804 b32 805 beamformer_set_live_parameters(BeamformerLiveImagingParameters *new) 806 { 807 b32 result = 0; 808 if (check_shared_memory()) { 809 memory_copy(&g_beamformer_library_context.bp->live_imaging_parameters, new, sizeof(*new)); 810 store_fence(); 811 result = 1; 812 } 813 return result; 814 }