beamformer_shared_memory.c (15884B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_SHARED_MEMORY_VERSION (38UL) 3 4 #define BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE KB(32) 5 6 #define BEAMFORMER_RF_ALIGN 256 7 8 #define BEAMFORMER_HANDLE(kind) struct kind##_T; typedef struct kind##_T *kind 9 10 BEAMFORMER_HANDLE(BeamformerSharedMemoryAllocationHandle); 11 12 typedef enum { 13 BeamformerSharedMemoryHandleKind_Nil, 14 BeamformerSharedMemoryHandleKind_Memory, 15 BeamformerSharedMemoryHandleKind_Count, 16 } BeamformerSharedMemoryHandleKind; 17 18 typedef struct { 19 u64 size; 20 u32 align; 21 u32 page_count; 22 } BeamformerSharedMemoryAllocation; 23 24 typedef struct { 25 u32 page_count; 26 u32 prev_page; 27 u32 next_page; 28 } BeamformerFreePageRun; 29 30 typedef enum { 31 BeamformerWorkKind_Compute, 32 BeamformerWorkKind_WaitThenCompute, 33 BeamformerWorkKind_CreateFilter, 34 BeamformerWorkKind_ExportBuffer, 35 } BeamformerWorkKind; 36 37 typedef struct { 38 BeamformerFilterParameters parameters; 39 u8 filter_slot; 40 u8 parameter_block; 41 static_assert(BeamformerFilterSlots <= 255, "CreateFilterContext only supports 255 filter slots"); 42 static_assert(BeamformerMaxParameterBlocks <= 255, "CreateFilterContext only supports 255 parameter blocks"); 43 } BeamformerCreateFilterContext; 44 45 typedef enum { 46 BeamformerExportKind_BeamformedData, 47 BeamformerExportKind_Stats, 48 } BeamformerExportKind; 49 50 typedef struct { 51 BeamformerExportKind kind; 52 u32 count; /* Number of frames to export */ 53 BeamformerSharedMemoryAllocationHandle memory; 54 } BeamformerExportContext; 55 56 #define BEAMFORMER_SHARED_MEMORY_LOCKS \ 57 X(UploadRF) \ 58 X(ExportSync) \ 59 X(DispatchCompute) 60 61 #define X(name) BeamformerSharedMemoryLockKind_##name, 62 typedef enum {BEAMFORMER_SHARED_MEMORY_LOCKS BeamformerSharedMemoryLockKind_Count} BeamformerSharedMemoryLockKind; 63 #undef X 64 65 typedef struct { 66 m4 voxel_transform; 67 u64 rf_offset; 68 f32 focus_depth; 69 f32 steering_angle; 70 u16 tx_rx_orientation; 71 u16 parameter_set; 72 BeamformerViewPlaneTag view_plane; 73 74 static_assert(BeamformerMaxParameterBlocks <= U16_MAX, ""); 75 } BeamformerComputeWorkContext; 76 77 /* NOTE: discriminated union based on type */ 78 typedef struct { 79 BeamformerWorkKind kind; 80 BeamformerSharedMemoryLockKind lock; 81 union { 82 BeamformerComputeWorkContext compute_context; 83 BeamformerCreateFilterContext create_filter_context; 84 BeamformerExportContext export_context; 85 BeamformerShaderKind reload_shader; 86 }; 87 } BeamformWork; 88 89 typedef struct { 90 union { 91 u64 queue; 92 struct {u32 widx, ridx;}; 93 }; 94 BeamformWork work_items[1 << 6]; 95 } BeamformWorkQueue; 96 97 #define BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \ 98 X(ComputePipeline, pipeline) \ 99 X(ChannelMapping, channel_mapping) \ 100 X(FocalVectors, focal_vectors) \ 101 X(Parameters, parameters) \ 102 X(SparseElements, sparse_elements) \ 103 X(TransmitReceiveOrientations, transmit_receive_orientations) \ 104 105 typedef enum { 106 #define X(k, ...) BeamformerParameterBlockRegion_##k, 107 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 108 #undef X 109 BeamformerParameterBlockRegion_Count 110 } BeamformerParameterBlockRegions; 111 112 typedef enum { 113 BeamformerParameterDirtyFlag_Parameters, 114 BeamformerParameterDirtyFlag_NotifyUI, 115 BeamformerParameterDirtyFlag_Count, 116 } BeamformerParameterDirtyFlags; 117 118 typedef union { 119 u8 filter_slot; 120 } BeamformerShaderParameters; 121 122 typedef struct { 123 BeamformerShaderKind shaders[BeamformerMaxComputeShaderStages]; 124 BeamformerShaderParameters parameters[BeamformerMaxComputeShaderStages]; 125 u32 shader_count; 126 BeamformerDataKind data_kind; 127 } BeamformerComputePipeline; 128 129 typedef struct { 130 alignas(16) BeamformerParameters parameters; 131 132 /* NOTE(rnp): signals to the beamformer that a subregion of a block has been updated */ 133 u32 region_update_flags; 134 static_assert(BeamformerParameterDirtyFlag_Count <= 32, ""); 135 136 BeamformerComputePipeline pipeline; 137 138 alignas(16) i16 channel_mapping[BeamformerMaxChannelCount]; 139 alignas(16) i16 sparse_elements[BeamformerMaxChannelCount]; 140 alignas(16) u8 transmit_receive_orientations[BeamformerMaxChannelCount]; 141 /* NOTE(rnp): interleaved transmit angle, focal depth pairs */ 142 alignas(16) v2 focal_vectors[BeamformerMaxChannelCount]; 143 } BeamformerParameterBlock; 144 static_assert(sizeof(BeamformerParameterBlock) % alignof(BeamformerParameterBlock) == 0, 145 "sizeof(BeamformerParametersBlock) must be a multiple of its alignment"); 146 147 #define X(k, field) [BeamformerParameterBlockRegion_##k] = offsetof(BeamformerParameterBlock, field), 148 read_only u16 BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_Count] = { 149 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 150 }; 151 #undef X 152 153 typedef struct { 154 u32 version; 155 156 /* NOTE(rnp): causes future library calls to fail. 157 * see note in beamformer_invalidate_shared_memory() */ 158 b32 invalid; 159 160 /* NOTE(rnp): not used for locking on w32 but we can use these to peek at the status of 161 * the lock without leaving userspace. */ 162 i32 locks[(u32)BeamformerSharedMemoryLockKind_Count + (u32)BeamformerMaxParameterBlocks]; 163 164 /* NOTE(rnp): total number of parameter block regions the client has requested. 165 * used to calculate offset to scratch space and to track number of allocated 166 * semaphores on w32. Defaults to 1 but can be changed at runtime */ 167 u32 reserved_parameter_blocks; 168 169 /* TODO(rnp): this is really sucky. we need a better way to communicate this */ 170 BeamformerSharedMemoryAllocationHandle rf_handle; 171 172 // NOTE(rnp): currently this cannot be directly user readable. its interpretation 173 // requires beamformer implementation details 174 u64 beamformed_frame_buffer_size; 175 176 struct { 177 u64 max_rf_data_size; 178 b8 cuda; 179 b8 hilbert; 180 } capabilities; 181 182 BeamformerLiveImagingParameters live_imaging_parameters; 183 BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags; 184 185 struct { 186 u32 first_free_page; 187 u32 page_count; 188 u64 offset; 189 u64 lock; 190 } transient; 191 192 BeamformWorkQueue external_work_queue; 193 } BeamformerSharedMemory; 194 195 function BeamformWork * 196 beamform_work_queue_pop(BeamformWorkQueue *q) 197 { 198 BeamformWork *result = 0; 199 200 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 201 u64 val = atomic_load_u64(&q->queue); 202 u64 mask = countof(q->work_items) - 1; 203 u64 widx = val & mask; 204 u64 ridx = val >> 32 & mask; 205 206 if (ridx != widx) 207 result = q->work_items + ridx; 208 209 return result; 210 } 211 212 function void 213 beamform_work_queue_pop_commit(BeamformWorkQueue *q) 214 { 215 atomic_add_u64(&q->queue, 0x100000000ULL); 216 } 217 218 function BeamformWork * 219 beamform_work_queue_push(BeamformWorkQueue *q) 220 { 221 BeamformWork *result = 0; 222 223 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 224 u64 val = atomic_load_u64(&q->queue); 225 u64 mask = countof(q->work_items) - 1; 226 u64 widx = val & mask; 227 u64 ridx = val >> 32 & mask; 228 u64 next = (widx + 1) & mask; 229 230 if (val & 0x80000000) 231 atomic_and_u64(&q->queue, ~0x80000000); 232 233 if (next != ridx) { 234 result = q->work_items + widx; 235 zero_struct(result); 236 } 237 238 return result; 239 } 240 241 function void 242 beamform_work_queue_push_commit(BeamformWorkQueue *q) 243 { 244 atomic_add_u64(&q->queue, 1); 245 } 246 247 #if OS_WINDOWS 248 // NOTE(rnp): junk needed on w32 to watch a value across processes while yielding 249 // control back to the kernel. There are user level CPU instructions that allow 250 // this so why w32 can't do it in kernel mode sounds like shitty design to me. 251 DEBUG_IMPORT OSW32Semaphore os_w32_shared_memory_semaphores[countof(((BeamformerSharedMemory *)0)->locks)]; 252 #endif 253 254 function b32 255 beamformer_shared_memory_take_lock(BeamformerSharedMemory *sm, i32 lock, u32 timeout_ms) 256 { 257 #if OS_WINDOWS 258 b32 result = os_w32_semaphore_wait(os_w32_shared_memory_semaphores[lock], timeout_ms); 259 if (result) atomic_store_u32(sm->locks + lock, 1); 260 #else 261 b32 result = take_lock(sm->locks + lock, timeout_ms); 262 #endif 263 return result; 264 } 265 266 function void 267 beamformer_shared_memory_release_lock(BeamformerSharedMemory *sm, i32 lock) 268 { 269 release_lock(sm->locks + lock); 270 #if OS_WINDOWS 271 os_w32_semaphore_release(os_w32_shared_memory_semaphores[lock], 1); 272 #endif 273 } 274 275 function BeamformerParameterBlock * 276 beamformer_parameter_block(BeamformerSharedMemory *sm, u32 block) 277 { 278 assert(sm->reserved_parameter_blocks > block); 279 BeamformerParameterBlock *result = (typeof(result))((u8 *)(sm + 1) + block * sizeof(*result)); 280 return result; 281 } 282 283 function b32 284 beamformer_parameter_block_dirty(BeamformerSharedMemory *sm, u32 block) 285 { 286 b32 result = beamformer_parameter_block(sm, block)->region_update_flags != 0; 287 return result; 288 } 289 290 function BeamformerParameterBlock * 291 beamformer_parameter_block_lock(BeamformerSharedMemory *sm, u32 block, i32 timeout_ms) 292 { 293 assert(block < BeamformerMaxParameterBlocks); 294 BeamformerParameterBlock *result = 0; 295 if (beamformer_shared_memory_take_lock(sm, BeamformerSharedMemoryLockKind_Count + block, (u32)timeout_ms)) 296 result = beamformer_parameter_block(sm, block); 297 return result; 298 } 299 300 function void 301 beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block) 302 { 303 assert(block < BeamformerMaxParameterBlocks); 304 beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block); 305 } 306 307 function void 308 beamformed_shared_memory_dirty_flag(BeamformerSharedMemory *sm, u32 block, BeamformerParameterDirtyFlags flag) 309 { 310 BeamformerParameterBlock *pb = beamformer_parameter_block(sm, block); 311 atomic_or_u32(&pb->region_update_flags, 1u << flag); 312 } 313 314 function void 315 post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind lock) 316 { 317 /* NOTE(rnp): debug: here it is not a bug to release the lock if it 318 * isn't held but elswhere it is */ 319 DEBUG_DECL(if (sm->locks[lock])) { 320 beamformer_shared_memory_release_lock(sm, lock); 321 } 322 } 323 324 function u32 325 beamformer_shared_memory_release_pages(u8 *base, u32 first_free_page, u32 page, u32 count, u32 page_size) 326 { 327 u32 current = first_free_page; 328 BeamformerFreePageRun *pr = (void *)(base + page_size * current); 329 while (current < page && page > pr->next_page) { 330 current = pr->next_page; 331 pr = (void *)(base + page_size * current); 332 } 333 334 u32 result = first_free_page; 335 336 BeamformerFreePageRun *to_free = (void *)(base + page_size * page); 337 zero_struct(to_free); 338 339 if (current > page) { 340 assert(current == first_free_page); 341 assert(pr->prev_page == -1u); 342 343 result = page; 344 345 to_free->page_count = count; 346 if (page + count == current) { 347 to_free->page_count += pr->page_count; 348 to_free->next_page = pr->next_page; 349 } else { 350 to_free->next_page = current; 351 pr->prev_page = page; 352 } 353 } else { 354 BeamformerFreePageRun *next = (void *)(base + page_size * pr->next_page); 355 if (current + pr->page_count == page) { 356 pr->page_count += count; 357 if (current + pr->page_count == pr->next_page) { 358 pr->next_page = next->next_page; 359 pr->page_count += next->page_count; 360 if (next->next_page != -1u) { 361 next = (void *)(base + page_size * next->next_page); 362 next->prev_page = current; 363 } 364 } 365 } else { 366 if (pr->next_page != -1u) next->prev_page = page; 367 to_free->next_page = pr->next_page; 368 to_free->page_count = count; 369 to_free->prev_page = current; 370 } 371 } 372 return result; 373 } 374 375 function void * 376 beamformer_shared_memory_make_handle(BeamformerSharedMemoryHandleKind kind, u64 payload) 377 { 378 static_assert(BeamformerSharedMemoryHandleKind_Count <= U16_MAX, ""); 379 u64 result = ((u64)kind << 48ul) | (payload & 0x0000fffffffffffful); 380 return (void *)result; 381 } 382 383 function BeamformerSharedMemoryHandleKind 384 beamformer_shared_memory_handle_kind(void *handle) 385 { 386 BeamformerSharedMemoryHandleKind result = (BeamformerSharedMemoryHandleKind)((u64)handle >> 48ul); 387 return result; 388 } 389 390 function u64 391 beamformer_shared_memory_handle_payload(void *handle) 392 { 393 u64 result = ((u64)handle & 0x0000fffffffffffful); 394 return result; 395 } 396 397 #define beamformer_shared_memory_transient_alloc(sm, ...) beamformer_shared_memory_transient_alloc_(sm, (ArenaAllocateInfo){.align = 8, .count = 1, __VA_ARGS__}) 398 function BeamformerSharedMemoryAllocationHandle 399 beamformer_shared_memory_transient_alloc_(BeamformerSharedMemory *sm, ArenaAllocateInfo info) 400 { 401 u64 expected = 0; 402 spin_while_false(atomic_cas_u64(&sm->transient.lock, &expected, 1)); 403 404 u64 needed_size = AlignUpPowerOfTwo(info.size * info.count + sizeof(BeamformerSharedMemoryAllocation), info.align); 405 u64 page_count = DivRoundUp(needed_size, BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 406 407 u8 *base = (u8 *)sm + sm->transient.offset; 408 BeamformerFreePageRun *pr = sm->transient.first_free_page != -1u ? (void *)(base + sm->transient.first_free_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE) : 0; 409 while (pr) { 410 if (pr->page_count >= page_count) 411 break; 412 pr = pr->next_page != -1u ? (void *)(base + pr->next_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE) : 0; 413 } 414 415 BeamformerSharedMemoryAllocationHandle result = 0; 416 if (pr) { 417 u64 new_free_page = (u64)((u8 *)pr - base) / BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE + page_count; 418 419 BeamformerFreePageRun *next = (void *)(base + new_free_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 420 next->page_count = pr->page_count - page_count; 421 next->prev_page = pr->prev_page; 422 next->next_page = pr->next_page; 423 424 if (next->prev_page != -1u) { 425 BeamformerFreePageRun *prev = (void *)(base + next->prev_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 426 prev->next_page = new_free_page; 427 } 428 429 if (next->next_page != -1u) { 430 BeamformerFreePageRun *nn = (void *)(base + next->next_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 431 nn->prev_page = new_free_page; 432 } 433 434 zero_struct(pr); 435 436 BeamformerSharedMemoryAllocation *ma = (void *)pr; 437 ma->size = info.size * info.count; 438 ma->align = info.align; 439 ma->page_count = page_count; 440 441 sm->transient.first_free_page = new_free_page; 442 443 result = beamformer_shared_memory_make_handle(BeamformerSharedMemoryHandleKind_Memory, 444 new_free_page - page_count); 445 } 446 atomic_store_u64(&sm->transient.lock, 0); 447 448 return result; 449 } 450 451 function void 452 beamformer_shared_memory_allocation_release(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle) 453 { 454 assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory); 455 456 u64 expected = 0; 457 spin_while_false(atomic_cas_u64(&sm->transient.lock, &expected, 1)); 458 { 459 u64 page = beamformer_shared_memory_handle_payload(handle); 460 u8 *base = (u8 *)sm + sm->transient.offset; 461 BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 462 u32 next_free = beamformer_shared_memory_release_pages(base, sm->transient.first_free_page, 463 page, ma->page_count, 464 BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 465 sm->transient.first_free_page = next_free; 466 } 467 atomic_store_u64(&sm->transient.lock, 0); 468 } 469 470 function void * 471 beamformer_shared_memory_allocation_pointer(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle) 472 { 473 assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory); 474 u64 page = beamformer_shared_memory_handle_payload(handle); 475 u8 *base = (u8 *)sm + sm->transient.offset; 476 BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 477 void *result = (void *)AlignUpPowerOfTwo((u64)(ma + 1), (u64)ma->align); 478 return result; 479 } 480 481 function u64 482 beamformer_shared_memory_allocation_size(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle) 483 { 484 assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory); 485 u64 page = beamformer_shared_memory_handle_payload(handle); 486 487 u8 *base = (u8 *)sm + sm->transient.offset; 488 BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 489 u64 result = ma->size; 490 return result; 491 }