beamformer_shared_memory.c (9823B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_SHARED_MEMORY_VERSION (36UL) 3 4 typedef enum { 5 BeamformerWorkKind_Compute, 6 BeamformerWorkKind_WaitThenCompute, 7 BeamformerWorkKind_CreateFilter, 8 BeamformerWorkKind_ExportBuffer, 9 } BeamformerWorkKind; 10 11 typedef struct { 12 BeamformerFilterParameters parameters; 13 u8 filter_slot; 14 u8 parameter_block; 15 static_assert(BeamformerFilterSlots <= 255, "CreateFilterContext only supports 255 filter slots"); 16 static_assert(BeamformerMaxParameterBlocks <= 255, "CreateFilterContext only supports 255 parameter blocks"); 17 } BeamformerCreateFilterContext; 18 19 typedef enum { 20 BeamformerExportKind_BeamformedData, 21 BeamformerExportKind_Stats, 22 } BeamformerExportKind; 23 24 typedef struct { 25 BeamformerExportKind kind; 26 u32 count; /* Number of items to export */ 27 u64 size; /* Total expected size of the exported data */ 28 } BeamformerExportContext; 29 30 #define BEAMFORMER_SHARED_MEMORY_LOCKS \ 31 X(ScratchSpace) \ 32 X(UploadRF) \ 33 X(ExportSync) \ 34 X(DispatchCompute) 35 36 #define X(name) BeamformerSharedMemoryLockKind_##name, 37 typedef enum {BEAMFORMER_SHARED_MEMORY_LOCKS BeamformerSharedMemoryLockKind_Count} BeamformerSharedMemoryLockKind; 38 #undef X 39 40 typedef struct { 41 BeamformerViewPlaneTag view_plane; 42 u32 parameter_block; 43 u64 rf_offset; 44 } BeamformerComputeWorkContext; 45 46 /* NOTE: discriminated union based on type */ 47 typedef struct { 48 BeamformerWorkKind kind; 49 BeamformerSharedMemoryLockKind lock; 50 union { 51 BeamformerComputeWorkContext compute_context; 52 BeamformerCreateFilterContext create_filter_context; 53 BeamformerExportContext export_context; 54 BeamformerShaderKind reload_shader; 55 }; 56 } BeamformWork; 57 58 typedef struct { 59 union { 60 u64 queue; 61 struct {u32 widx, ridx;}; 62 }; 63 BeamformWork work_items[1 << 6]; 64 } BeamformWorkQueue; 65 66 #define BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \ 67 X(ComputePipeline, pipeline) \ 68 X(ChannelMapping, channel_mapping) \ 69 X(FocalVectors, focal_vectors) \ 70 X(Parameters, parameters) \ 71 X(SparseElements, sparse_elements) \ 72 X(TransmitReceiveOrientations, transmit_receive_orientations) \ 73 74 typedef enum { 75 #define X(k, ...) BeamformerParameterBlockRegion_##k, 76 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 77 #undef X 78 BeamformerParameterBlockRegion_Count 79 } BeamformerParameterBlockRegions; 80 81 typedef enum { 82 BeamformerParameterDirtyFlag_Parameters, 83 BeamformerParameterDirtyFlag_NotifyUI, 84 BeamformerParameterDirtyFlag_Count, 85 } BeamformerParameterDirtyFlags; 86 87 typedef union { 88 u8 filter_slot; 89 } BeamformerShaderParameters; 90 91 typedef struct { 92 BeamformerShaderKind shaders[BeamformerMaxComputeShaderStages]; 93 BeamformerShaderParameters parameters[BeamformerMaxComputeShaderStages]; 94 u32 shader_count; 95 BeamformerDataKind data_kind; 96 } BeamformerComputePipeline; 97 98 typedef struct { 99 alignas(16) union { 100 BeamformerParameters parameters; 101 struct { 102 BeamformerParametersHead parameters_head; 103 BeamformerUIParameters parameters_ui; 104 BeamformerExtraParameters parameters_extra; 105 }; 106 }; 107 108 /* NOTE(rnp): signals to the beamformer that a subregion of a block has been updated */ 109 u32 region_update_flags; 110 static_assert(BeamformerParameterDirtyFlag_Count <= 32, ""); 111 112 BeamformerComputePipeline pipeline; 113 114 alignas(16) i16 channel_mapping[BeamformerMaxChannelCount]; 115 alignas(16) i16 sparse_elements[BeamformerMaxChannelCount]; 116 alignas(16) u8 transmit_receive_orientations[BeamformerMaxChannelCount]; 117 /* NOTE(rnp): interleaved transmit angle, focal depth pairs */ 118 alignas(16) v2 focal_vectors[BeamformerMaxChannelCount]; 119 } BeamformerParameterBlock; 120 static_assert(sizeof(BeamformerParameterBlock) % alignof(BeamformerParameterBlock) == 0, 121 "sizeof(BeamformerParametersBlock) must be a multiple of its alignment"); 122 123 #define X(k, field) [BeamformerParameterBlockRegion_##k] = offsetof(BeamformerParameterBlock, field), 124 read_only u16 BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_Count] = { 125 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 126 }; 127 #undef X 128 129 typedef struct { 130 u32 version; 131 132 /* NOTE(rnp): causes future library calls to fail. 133 * see note in beamformer_invalidate_shared_memory() */ 134 b32 invalid; 135 136 /* NOTE(rnp): not used for locking on w32 but we can use these to peek at the status of 137 * the lock without leaving userspace. */ 138 i32 locks[(u32)BeamformerSharedMemoryLockKind_Count + (u32)BeamformerMaxParameterBlocks]; 139 140 /* NOTE(rnp): total number of parameter block regions the client has requested. 141 * used to calculate offset to scratch space and to track number of allocated 142 * semaphores on w32. Defaults to 1 but can be changed at runtime */ 143 u32 reserved_parameter_blocks; 144 145 /* TODO(rnp): this is really sucky. we need a better way to communicate this */ 146 u64 rf_block_rf_size; 147 148 b64 new_rf_upload; 149 150 // NOTE(rnp): currently this cannot be directly user readable. its interpretation 151 // requires beamformer implementation details 152 u64 beamformed_frame_buffer_size; 153 154 struct { 155 u64 max_rf_data_size; 156 b8 cuda; 157 b8 hilbert; 158 } capabilities; 159 160 BeamformerLiveImagingParameters live_imaging_parameters; 161 BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags; 162 163 BeamformWorkQueue external_work_queue; 164 } BeamformerSharedMemory; 165 166 function BeamformWork * 167 beamform_work_queue_pop(BeamformWorkQueue *q) 168 { 169 BeamformWork *result = 0; 170 171 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 172 u64 val = atomic_load_u64(&q->queue); 173 u64 mask = countof(q->work_items) - 1; 174 u64 widx = val & mask; 175 u64 ridx = val >> 32 & mask; 176 177 if (ridx != widx) 178 result = q->work_items + ridx; 179 180 return result; 181 } 182 183 function void 184 beamform_work_queue_pop_commit(BeamformWorkQueue *q) 185 { 186 atomic_add_u64(&q->queue, 0x100000000ULL); 187 } 188 189 function BeamformWork * 190 beamform_work_queue_push(BeamformWorkQueue *q) 191 { 192 BeamformWork *result = 0; 193 194 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 195 u64 val = atomic_load_u64(&q->queue); 196 u64 mask = countof(q->work_items) - 1; 197 u64 widx = val & mask; 198 u64 ridx = val >> 32 & mask; 199 u64 next = (widx + 1) & mask; 200 201 if (val & 0x80000000) 202 atomic_and_u64(&q->queue, ~0x80000000); 203 204 if (next != ridx) { 205 result = q->work_items + widx; 206 zero_struct(result); 207 } 208 209 return result; 210 } 211 212 function void 213 beamform_work_queue_push_commit(BeamformWorkQueue *q) 214 { 215 atomic_add_u64(&q->queue, 1); 216 } 217 218 #if OS_WINDOWS 219 // NOTE(rnp): junk needed on w32 to watch a value across processes while yielding 220 // control back to the kernel. There are user level CPU instructions that allow 221 // this so why w32 can't do it in kernel mode sounds like shitty design to me. 222 DEBUG_IMPORT OSW32Semaphore os_w32_shared_memory_semaphores[countof(((BeamformerSharedMemory *)0)->locks)]; 223 #endif 224 225 function b32 226 beamformer_shared_memory_take_lock(BeamformerSharedMemory *sm, i32 lock, u32 timeout_ms) 227 { 228 #if OS_WINDOWS 229 b32 result = os_w32_semaphore_wait(os_w32_shared_memory_semaphores[lock], timeout_ms); 230 if (result) atomic_store_u32(sm->locks + lock, 1); 231 #else 232 b32 result = take_lock(sm->locks + lock, timeout_ms); 233 #endif 234 return result; 235 } 236 237 function void 238 beamformer_shared_memory_release_lock(BeamformerSharedMemory *sm, i32 lock) 239 { 240 release_lock(sm->locks + lock); 241 #if OS_WINDOWS 242 os_w32_semaphore_release(os_w32_shared_memory_semaphores[lock], 1); 243 #endif 244 } 245 246 function BeamformerParameterBlock * 247 beamformer_parameter_block(BeamformerSharedMemory *sm, u32 block) 248 { 249 assert(sm->reserved_parameter_blocks > block); 250 BeamformerParameterBlock *result = (typeof(result))((u8 *)(sm + 1) + block * sizeof(*result)); 251 return result; 252 } 253 254 function b32 255 beamformer_parameter_block_dirty(BeamformerSharedMemory *sm, u32 block) 256 { 257 b32 result = beamformer_parameter_block(sm, block)->region_update_flags != 0; 258 return result; 259 } 260 261 function BeamformerParameterBlock * 262 beamformer_parameter_block_lock(BeamformerSharedMemory *sm, u32 block, i32 timeout_ms) 263 { 264 assert(block < BeamformerMaxParameterBlocks); 265 BeamformerParameterBlock *result = 0; 266 if (beamformer_shared_memory_take_lock(sm, BeamformerSharedMemoryLockKind_Count + block, (u32)timeout_ms)) 267 result = beamformer_parameter_block(sm, block); 268 return result; 269 } 270 271 function void 272 beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block) 273 { 274 assert(block < BeamformerMaxParameterBlocks); 275 beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block); 276 } 277 278 function Arena * 279 beamformer_shared_memory_scratch_arena(BeamformerSharedMemory *sm, i64 shared_memory_size) 280 { 281 assert(sm->reserved_parameter_blocks > 0); 282 BeamformerParameterBlock *last = beamformer_parameter_block(sm, sm->reserved_parameter_blocks - 1); 283 u64 size = (u64)((u8 *)sm + shared_memory_size - (u8 *)(last + 1)); 284 Arena *result = arena_create(.optional_backing_store = (last + 1), .reserve_size = size, .commit_size = size, .flags = ArenaFlag_NoChain); 285 arena_pre_align(result, KB(4)); 286 return result; 287 } 288 289 function void * 290 beamformer_shared_memory_data_pointer(BeamformerSharedMemory *sm, i64 shared_memory_size) 291 { 292 Arena *arena = beamformer_shared_memory_scratch_arena(sm, shared_memory_size); 293 void *result = (u8 *)arena + arena->position; 294 return result; 295 } 296 297 function void 298 beamformed_shared_memory_dirty_flag(BeamformerSharedMemory *sm, u32 block, BeamformerParameterDirtyFlags flag) 299 { 300 BeamformerParameterBlock *pb = beamformer_parameter_block(sm, block); 301 atomic_or_u32(&pb->region_update_flags, 1u << flag); 302 } 303 304 function void 305 post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind lock) 306 { 307 /* NOTE(rnp): debug: here it is not a bug to release the lock if it 308 * isn't held but elswhere it is */ 309 DEBUG_DECL(if (sm->locks[lock])) { 310 beamformer_shared_memory_release_lock(sm, lock); 311 } 312 }