beamformer_shared_memory.c (9774B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_SHARED_MEMORY_VERSION (35UL) 3 4 typedef enum { 5 BeamformerWorkKind_Compute, 6 BeamformerWorkKind_ComputeIndirect, 7 BeamformerWorkKind_CreateFilter, 8 BeamformerWorkKind_ExportBuffer, 9 } BeamformerWorkKind; 10 11 typedef struct { 12 BeamformerFilterParameters parameters; 13 u8 filter_slot; 14 u8 parameter_block; 15 static_assert(BeamformerFilterSlots <= 255, "CreateFilterContext only supports 255 filter slots"); 16 static_assert(BeamformerMaxParameterBlocks <= 255, "CreateFilterContext only supports 255 parameter blocks"); 17 } BeamformerCreateFilterContext; 18 19 typedef enum { 20 BeamformerExportKind_BeamformedData, 21 BeamformerExportKind_Stats, 22 } BeamformerExportKind; 23 24 typedef struct { 25 BeamformerExportKind kind; 26 u32 count; /* Number of items to export */ 27 u64 size; /* Total expected size of the exported data */ 28 } BeamformerExportContext; 29 30 #define BEAMFORMER_SHARED_MEMORY_LOCKS \ 31 X(ScratchSpace) \ 32 X(UploadRF) \ 33 X(ExportSync) \ 34 X(DispatchCompute) 35 36 #define X(name) BeamformerSharedMemoryLockKind_##name, 37 typedef enum {BEAMFORMER_SHARED_MEMORY_LOCKS BeamformerSharedMemoryLockKind_Count} BeamformerSharedMemoryLockKind; 38 #undef X 39 40 typedef struct { 41 BeamformerViewPlaneTag view_plane; 42 u32 parameter_block; 43 } BeamformerComputeWorkContext; 44 45 /* NOTE: discriminated union based on type */ 46 typedef struct { 47 BeamformerWorkKind kind; 48 BeamformerSharedMemoryLockKind lock; 49 union { 50 BeamformerComputeWorkContext compute_context; 51 BeamformerCreateFilterContext create_filter_context; 52 BeamformerExportContext export_context; 53 BeamformerShaderKind reload_shader; 54 }; 55 } BeamformWork; 56 57 typedef struct { 58 union { 59 u64 queue; 60 struct {u32 widx, ridx;}; 61 }; 62 BeamformWork work_items[1 << 6]; 63 } BeamformWorkQueue; 64 65 #define BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \ 66 X(ComputePipeline, pipeline) \ 67 X(ChannelMapping, channel_mapping) \ 68 X(FocalVectors, focal_vectors) \ 69 X(Parameters, parameters) \ 70 X(SparseElements, sparse_elements) \ 71 X(TransmitReceiveOrientations, transmit_receive_orientations) \ 72 73 typedef enum { 74 #define X(k, ...) BeamformerParameterBlockRegion_##k, 75 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 76 #undef X 77 BeamformerParameterBlockRegion_Count 78 } BeamformerParameterBlockRegions; 79 80 typedef enum { 81 BeamformerParameterDirtyFlag_Parameters, 82 BeamformerParameterDirtyFlag_NotifyUI, 83 BeamformerParameterDirtyFlag_Count, 84 } BeamformerParameterDirtyFlags; 85 86 typedef union { 87 u8 filter_slot; 88 } BeamformerShaderParameters; 89 90 typedef struct { 91 BeamformerShaderKind shaders[BeamformerMaxComputeShaderStages]; 92 BeamformerShaderParameters parameters[BeamformerMaxComputeShaderStages]; 93 u32 shader_count; 94 BeamformerDataKind data_kind; 95 } BeamformerComputePipeline; 96 97 typedef struct { 98 alignas(16) union { 99 BeamformerParameters parameters; 100 struct { 101 BeamformerParametersHead parameters_head; 102 BeamformerUIParameters parameters_ui; 103 BeamformerExtraParameters parameters_extra; 104 }; 105 }; 106 107 /* NOTE(rnp): signals to the beamformer that a subregion of a block has been updated */ 108 u32 region_update_flags; 109 static_assert(BeamformerParameterDirtyFlag_Count <= 32, ""); 110 111 BeamformerComputePipeline pipeline; 112 113 alignas(16) i16 channel_mapping[BeamformerMaxChannelCount]; 114 alignas(16) i16 sparse_elements[BeamformerMaxChannelCount]; 115 alignas(16) u8 transmit_receive_orientations[BeamformerMaxChannelCount]; 116 /* NOTE(rnp): interleaved transmit angle, focal depth pairs */ 117 alignas(16) v2 focal_vectors[BeamformerMaxChannelCount]; 118 } BeamformerParameterBlock; 119 static_assert(sizeof(BeamformerParameterBlock) % alignof(BeamformerParameterBlock) == 0, 120 "sizeof(BeamformerParametersBlock) must be a multiple of its alignment"); 121 122 #define X(k, field) [BeamformerParameterBlockRegion_##k] = offsetof(BeamformerParameterBlock, field), 123 read_only global u16 BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_Count] = { 124 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 125 }; 126 #undef X 127 128 typedef struct { 129 u32 version; 130 131 /* NOTE(rnp): causes future library calls to fail. 132 * see note in beamformer_invalidate_shared_memory() */ 133 b32 invalid; 134 135 /* NOTE(rnp): not used for locking on w32 but we can use these to peek at the status of 136 * the lock without leaving userspace. */ 137 i32 locks[(u32)BeamformerSharedMemoryLockKind_Count + (u32)BeamformerMaxParameterBlocks]; 138 139 /* NOTE(rnp): total number of parameter block regions the client has requested. 140 * used to calculate offset to scratch space and to track number of allocated 141 * semaphores on w32. Defaults to 1 but can be changed at runtime */ 142 u32 reserved_parameter_blocks; 143 144 /* TODO(rnp): this is really sucky. we need a better way to communicate this */ 145 u64 rf_block_rf_size; 146 147 // NOTE(rnp): currently this cannot be directly user readable. its interpretation 148 // requires beamformer implementation details 149 u64 beamformed_frame_buffer_size; 150 151 struct { 152 u64 max_rf_data_size; 153 b8 cuda; 154 b8 hilbert; 155 } capabilities; 156 157 BeamformerLiveImagingParameters live_imaging_parameters; 158 BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags; 159 160 BeamformWorkQueue external_work_queue; 161 } BeamformerSharedMemory; 162 163 function BeamformWork * 164 beamform_work_queue_pop(BeamformWorkQueue *q) 165 { 166 BeamformWork *result = 0; 167 168 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 169 u64 val = atomic_load_u64(&q->queue); 170 u64 mask = countof(q->work_items) - 1; 171 u64 widx = val & mask; 172 u64 ridx = val >> 32 & mask; 173 174 if (ridx != widx) 175 result = q->work_items + ridx; 176 177 return result; 178 } 179 180 function void 181 beamform_work_queue_pop_commit(BeamformWorkQueue *q) 182 { 183 atomic_add_u64(&q->queue, 0x100000000ULL); 184 } 185 186 function BeamformWork * 187 beamform_work_queue_push(BeamformWorkQueue *q) 188 { 189 BeamformWork *result = 0; 190 191 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 192 u64 val = atomic_load_u64(&q->queue); 193 u64 mask = countof(q->work_items) - 1; 194 u64 widx = val & mask; 195 u64 ridx = val >> 32 & mask; 196 u64 next = (widx + 1) & mask; 197 198 if (val & 0x80000000) 199 atomic_and_u64(&q->queue, ~0x80000000); 200 201 if (next != ridx) { 202 result = q->work_items + widx; 203 zero_struct(result); 204 } 205 206 return result; 207 } 208 209 function void 210 beamform_work_queue_push_commit(BeamformWorkQueue *q) 211 { 212 atomic_add_u64(&q->queue, 1); 213 } 214 215 #if OS_WINDOWS 216 // NOTE(rnp): junk needed on w32 to watch a value across processes while yielding 217 // control back to the kernel. There are user level CPU instructions that allow 218 // this so why w32 can't do it in kernel mode sounds like shitty design to me. 219 DEBUG_IMPORT OSW32Semaphore os_w32_shared_memory_semaphores[countof(((BeamformerSharedMemory *)0)->locks)]; 220 #endif 221 222 function b32 223 beamformer_shared_memory_take_lock(BeamformerSharedMemory *sm, i32 lock, u32 timeout_ms) 224 { 225 #if OS_WINDOWS 226 b32 result = os_w32_semaphore_wait(os_w32_shared_memory_semaphores[lock], timeout_ms); 227 if (result) atomic_store_u32(sm->locks + lock, 1); 228 #else 229 b32 result = take_lock(sm->locks + lock, timeout_ms); 230 #endif 231 return result; 232 } 233 234 function void 235 beamformer_shared_memory_release_lock(BeamformerSharedMemory *sm, i32 lock) 236 { 237 release_lock(sm->locks + lock); 238 #if OS_WINDOWS 239 os_w32_semaphore_release(os_w32_shared_memory_semaphores[lock], 1); 240 #endif 241 } 242 243 function BeamformerParameterBlock * 244 beamformer_parameter_block(BeamformerSharedMemory *sm, u32 block) 245 { 246 assert(sm->reserved_parameter_blocks > block); 247 BeamformerParameterBlock *result = (typeof(result))((u8 *)(sm + 1) + block * sizeof(*result)); 248 return result; 249 } 250 251 function b32 252 beamformer_parameter_block_dirty(BeamformerSharedMemory *sm, u32 block) 253 { 254 b32 result = beamformer_parameter_block(sm, block)->region_update_flags != 0; 255 return result; 256 } 257 258 function BeamformerParameterBlock * 259 beamformer_parameter_block_lock(BeamformerSharedMemory *sm, u32 block, i32 timeout_ms) 260 { 261 assert(block < BeamformerMaxParameterBlocks); 262 BeamformerParameterBlock *result = 0; 263 if (beamformer_shared_memory_take_lock(sm, BeamformerSharedMemoryLockKind_Count + block, (u32)timeout_ms)) 264 result = beamformer_parameter_block(sm, block); 265 return result; 266 } 267 268 function void 269 beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block) 270 { 271 assert(block < BeamformerMaxParameterBlocks); 272 beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block); 273 } 274 275 function Arena * 276 beamformer_shared_memory_scratch_arena(BeamformerSharedMemory *sm, i64 shared_memory_size) 277 { 278 assert(sm->reserved_parameter_blocks > 0); 279 BeamformerParameterBlock *last = beamformer_parameter_block(sm, sm->reserved_parameter_blocks - 1); 280 u64 size = (u64)((u8 *)sm + shared_memory_size - (u8 *)(last + 1)); 281 Arena *result = arena_create(.optional_backing_store = (last + 1), .reserve_size = size, .commit_size = size, .flags = ArenaFlag_NoChain); 282 arena_pre_align(result, KB(4)); 283 return result; 284 } 285 286 function void * 287 beamformer_shared_memory_data_pointer(BeamformerSharedMemory *sm, i64 shared_memory_size) 288 { 289 Arena *arena = beamformer_shared_memory_scratch_arena(sm, shared_memory_size); 290 void *result = (u8 *)arena + arena->position; 291 return result; 292 } 293 294 function void 295 beamformed_shared_memory_dirty_flag(BeamformerSharedMemory *sm, u32 block, BeamformerParameterDirtyFlags flag) 296 { 297 BeamformerParameterBlock *pb = beamformer_parameter_block(sm, block); 298 atomic_or_u32(&pb->region_update_flags, 1u << flag); 299 } 300 301 function void 302 post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind lock) 303 { 304 /* NOTE(rnp): debug: here it is not a bug to release the lock if it 305 * isn't held but elswhere it is */ 306 DEBUG_DECL(if (sm->locks[lock])) { 307 beamformer_shared_memory_release_lock(sm, lock); 308 } 309 }