ogl_beamforming

Ultrasound Beamforming Implemented with OpenGL
git clone anongit@rnpnr.xyz:ogl_beamforming.git
Log | Files | Refs | Feed | Submodules | README | LICENSE

beamformer_shared_memory.c (15884B)


      1 /* See LICENSE for license details. */
      2 #define BEAMFORMER_SHARED_MEMORY_VERSION (38UL)
      3 
      4 #define BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE KB(32)
      5 
      6 #define BEAMFORMER_RF_ALIGN 256
      7 
      8 #define BEAMFORMER_HANDLE(kind) struct kind##_T; typedef struct kind##_T *kind
      9 
     10 BEAMFORMER_HANDLE(BeamformerSharedMemoryAllocationHandle);
     11 
     12 typedef enum {
     13 	BeamformerSharedMemoryHandleKind_Nil,
     14 	BeamformerSharedMemoryHandleKind_Memory,
     15 	BeamformerSharedMemoryHandleKind_Count,
     16 } BeamformerSharedMemoryHandleKind;
     17 
     18 typedef struct {
     19 	u64 size;
     20 	u32 align;
     21 	u32 page_count;
     22 } BeamformerSharedMemoryAllocation;
     23 
     24 typedef struct {
     25 	u32 page_count;
     26 	u32 prev_page;
     27 	u32 next_page;
     28 } BeamformerFreePageRun;
     29 
     30 typedef enum {
     31 	BeamformerWorkKind_Compute,
     32 	BeamformerWorkKind_WaitThenCompute,
     33 	BeamformerWorkKind_CreateFilter,
     34 	BeamformerWorkKind_ExportBuffer,
     35 } BeamformerWorkKind;
     36 
     37 typedef struct {
     38 	BeamformerFilterParameters parameters;
     39 	u8 filter_slot;
     40 	u8 parameter_block;
     41 	static_assert(BeamformerFilterSlots        <= 255, "CreateFilterContext only supports 255 filter slots");
     42 	static_assert(BeamformerMaxParameterBlocks <= 255, "CreateFilterContext only supports 255 parameter blocks");
     43 } BeamformerCreateFilterContext;
     44 
     45 typedef enum {
     46 	BeamformerExportKind_BeamformedData,
     47 	BeamformerExportKind_Stats,
     48 } BeamformerExportKind;
     49 
     50 typedef struct {
     51 	BeamformerExportKind kind;
     52 	u32 count; /* Number of frames to export */
     53 	BeamformerSharedMemoryAllocationHandle memory;
     54 } BeamformerExportContext;
     55 
     56 #define BEAMFORMER_SHARED_MEMORY_LOCKS \
     57 	X(UploadRF)        \
     58 	X(ExportSync)      \
     59 	X(DispatchCompute)
     60 
     61 #define X(name) BeamformerSharedMemoryLockKind_##name,
     62 typedef enum {BEAMFORMER_SHARED_MEMORY_LOCKS BeamformerSharedMemoryLockKind_Count} BeamformerSharedMemoryLockKind;
     63 #undef X
     64 
     65 typedef struct {
     66 	m4  voxel_transform;
     67 	u64 rf_offset;
     68 	f32 focus_depth;
     69 	f32 steering_angle;
     70 	u16 tx_rx_orientation;
     71 	u16 parameter_set;
     72 	BeamformerViewPlaneTag view_plane;
     73 
     74 	static_assert(BeamformerMaxParameterBlocks <= U16_MAX, "");
     75 } BeamformerComputeWorkContext;
     76 
     77 /* NOTE: discriminated union based on type */
     78 typedef struct {
     79 	BeamformerWorkKind kind;
     80 	BeamformerSharedMemoryLockKind lock;
     81 	union {
     82 		BeamformerComputeWorkContext  compute_context;
     83 		BeamformerCreateFilterContext create_filter_context;
     84 		BeamformerExportContext       export_context;
     85 		BeamformerShaderKind          reload_shader;
     86 	};
     87 } BeamformWork;
     88 
     89 typedef struct {
     90 	union {
     91 		u64 queue;
     92 		struct {u32 widx, ridx;};
     93 	};
     94 	BeamformWork work_items[1 << 6];
     95 } BeamformWorkQueue;
     96 
     97 #define BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \
     98 	X(ComputePipeline,             pipeline)        \
     99 	X(ChannelMapping,              channel_mapping) \
    100 	X(FocalVectors,                focal_vectors)   \
    101 	X(Parameters,                  parameters)      \
    102 	X(SparseElements,              sparse_elements) \
    103 	X(TransmitReceiveOrientations, transmit_receive_orientations) \
    104 
    105 typedef enum {
    106 	#define X(k, ...) BeamformerParameterBlockRegion_##k,
    107 	BEAMFORMER_PARAMETER_BLOCK_REGION_LIST
    108 	#undef X
    109 	BeamformerParameterBlockRegion_Count
    110 } BeamformerParameterBlockRegions;
    111 
    112 typedef enum {
    113 	BeamformerParameterDirtyFlag_Parameters,
    114 	BeamformerParameterDirtyFlag_NotifyUI,
    115 	BeamformerParameterDirtyFlag_Count,
    116 } BeamformerParameterDirtyFlags;
    117 
    118 typedef union {
    119 	u8 filter_slot;
    120 } BeamformerShaderParameters;
    121 
    122 typedef struct {
    123 	BeamformerShaderKind       shaders[BeamformerMaxComputeShaderStages];
    124 	BeamformerShaderParameters parameters[BeamformerMaxComputeShaderStages];
    125 	u32                        shader_count;
    126 	BeamformerDataKind         data_kind;
    127 } BeamformerComputePipeline;
    128 
    129 typedef struct {
    130 	alignas(16) BeamformerParameters parameters;
    131 
    132 	/* NOTE(rnp): signals to the beamformer that a subregion of a block has been updated */
    133 	u32 region_update_flags;
    134 	static_assert(BeamformerParameterDirtyFlag_Count <= 32, "");
    135 
    136 	BeamformerComputePipeline pipeline;
    137 
    138 	alignas(16) i16 channel_mapping[BeamformerMaxChannelCount];
    139 	alignas(16) i16 sparse_elements[BeamformerMaxChannelCount];
    140 	alignas(16) u8  transmit_receive_orientations[BeamformerMaxChannelCount];
    141 	/* NOTE(rnp): interleaved transmit angle, focal depth pairs */
    142 	alignas(16) v2  focal_vectors[BeamformerMaxChannelCount];
    143 } BeamformerParameterBlock;
    144 static_assert(sizeof(BeamformerParameterBlock) % alignof(BeamformerParameterBlock) == 0,
    145               "sizeof(BeamformerParametersBlock) must be a multiple of its alignment");
    146 
    147 #define X(k, field) [BeamformerParameterBlockRegion_##k] = offsetof(BeamformerParameterBlock, field),
    148 read_only u16 BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_Count] = {
    149 	BEAMFORMER_PARAMETER_BLOCK_REGION_LIST
    150 };
    151 #undef X
    152 
    153 typedef struct {
    154 	u32 version;
    155 
    156 	/* NOTE(rnp): causes future library calls to fail.
    157 	 * see note in beamformer_invalidate_shared_memory() */
    158 	b32 invalid;
    159 
    160 	/* NOTE(rnp): not used for locking on w32 but we can use these to peek at the status of
    161 	 * the lock without leaving userspace. */
    162 	i32 locks[(u32)BeamformerSharedMemoryLockKind_Count + (u32)BeamformerMaxParameterBlocks];
    163 
    164 	/* NOTE(rnp): total number of parameter block regions the client has requested.
    165 	 * used to calculate offset to scratch space and to track number of allocated
    166 	 * semaphores on w32. Defaults to 1 but can be changed at runtime */
    167 	u32 reserved_parameter_blocks;
    168 
    169 	/* TODO(rnp): this is really sucky. we need a better way to communicate this */
    170 	BeamformerSharedMemoryAllocationHandle rf_handle;
    171 
    172 	// NOTE(rnp): currently this cannot be directly user readable. its interpretation
    173 	// requires beamformer implementation details
    174 	u64 beamformed_frame_buffer_size;
    175 
    176 	struct {
    177 		u64 max_rf_data_size;
    178 		b8  cuda;
    179 		b8  hilbert;
    180 	} capabilities;
    181 
    182 	BeamformerLiveImagingParameters live_imaging_parameters;
    183 	BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags;
    184 
    185 	struct {
    186 		u32 first_free_page;
    187 		u32 page_count;
    188 		u64 offset;
    189 		u64 lock;
    190 	} transient;
    191 
    192 	BeamformWorkQueue external_work_queue;
    193 } BeamformerSharedMemory;
    194 
    195 function BeamformWork *
    196 beamform_work_queue_pop(BeamformWorkQueue *q)
    197 {
    198 	BeamformWork *result = 0;
    199 
    200 	static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2");
    201 	u64 val  = atomic_load_u64(&q->queue);
    202 	u64 mask = countof(q->work_items) - 1;
    203 	u64 widx = val       & mask;
    204 	u64 ridx = val >> 32 & mask;
    205 
    206 	if (ridx != widx)
    207 		result = q->work_items + ridx;
    208 
    209 	return result;
    210 }
    211 
    212 function void
    213 beamform_work_queue_pop_commit(BeamformWorkQueue *q)
    214 {
    215 	atomic_add_u64(&q->queue, 0x100000000ULL);
    216 }
    217 
    218 function BeamformWork *
    219 beamform_work_queue_push(BeamformWorkQueue *q)
    220 {
    221 	BeamformWork *result = 0;
    222 
    223 	static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2");
    224 	u64 val  = atomic_load_u64(&q->queue);
    225 	u64 mask = countof(q->work_items) - 1;
    226 	u64 widx = val        & mask;
    227 	u64 ridx = val >> 32  & mask;
    228 	u64 next = (widx + 1) & mask;
    229 
    230 	if (val & 0x80000000)
    231 		atomic_and_u64(&q->queue, ~0x80000000);
    232 
    233 	if (next != ridx) {
    234 		result = q->work_items + widx;
    235 		zero_struct(result);
    236 	}
    237 
    238 	return result;
    239 }
    240 
    241 function void
    242 beamform_work_queue_push_commit(BeamformWorkQueue *q)
    243 {
    244 	atomic_add_u64(&q->queue, 1);
    245 }
    246 
    247 #if OS_WINDOWS
    248 // NOTE(rnp): junk needed on w32 to watch a value across processes while yielding
    249 // control back to the kernel. There are user level CPU instructions that allow
    250 // this so why w32 can't do it in kernel mode sounds like shitty design to me.
    251 DEBUG_IMPORT OSW32Semaphore os_w32_shared_memory_semaphores[countof(((BeamformerSharedMemory *)0)->locks)];
    252 #endif
    253 
    254 function b32
    255 beamformer_shared_memory_take_lock(BeamformerSharedMemory *sm, i32 lock, u32 timeout_ms)
    256 {
    257 #if OS_WINDOWS
    258 	b32 result = os_w32_semaphore_wait(os_w32_shared_memory_semaphores[lock], timeout_ms);
    259 	if (result) atomic_store_u32(sm->locks + lock, 1);
    260 #else
    261 	b32 result = take_lock(sm->locks + lock, timeout_ms);
    262 #endif
    263 	return result;
    264 }
    265 
    266 function void
    267 beamformer_shared_memory_release_lock(BeamformerSharedMemory *sm, i32 lock)
    268 {
    269 	release_lock(sm->locks + lock);
    270 #if OS_WINDOWS
    271 	os_w32_semaphore_release(os_w32_shared_memory_semaphores[lock], 1);
    272 #endif
    273 }
    274 
    275 function BeamformerParameterBlock *
    276 beamformer_parameter_block(BeamformerSharedMemory *sm, u32 block)
    277 {
    278 	assert(sm->reserved_parameter_blocks > block);
    279 	BeamformerParameterBlock *result = (typeof(result))((u8 *)(sm + 1) + block * sizeof(*result));
    280 	return result;
    281 }
    282 
    283 function b32
    284 beamformer_parameter_block_dirty(BeamformerSharedMemory *sm, u32 block)
    285 {
    286 	b32 result = beamformer_parameter_block(sm, block)->region_update_flags != 0;
    287 	return result;
    288 }
    289 
    290 function BeamformerParameterBlock *
    291 beamformer_parameter_block_lock(BeamformerSharedMemory *sm, u32 block, i32 timeout_ms)
    292 {
    293 	assert(block < BeamformerMaxParameterBlocks);
    294 	BeamformerParameterBlock *result = 0;
    295 	if (beamformer_shared_memory_take_lock(sm, BeamformerSharedMemoryLockKind_Count + block, (u32)timeout_ms))
    296 		result = beamformer_parameter_block(sm, block);
    297 	return result;
    298 }
    299 
    300 function void
    301 beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block)
    302 {
    303 	assert(block < BeamformerMaxParameterBlocks);
    304 	beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block);
    305 }
    306 
    307 function void
    308 beamformed_shared_memory_dirty_flag(BeamformerSharedMemory *sm, u32 block, BeamformerParameterDirtyFlags flag)
    309 {
    310 	BeamformerParameterBlock *pb = beamformer_parameter_block(sm, block);
    311 	atomic_or_u32(&pb->region_update_flags, 1u << flag);
    312 }
    313 
    314 function void
    315 post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind lock)
    316 {
    317 	/* NOTE(rnp): debug: here it is not a bug to release the lock if it
    318 	 * isn't held but elswhere it is */
    319 	DEBUG_DECL(if (sm->locks[lock])) {
    320 		beamformer_shared_memory_release_lock(sm, lock);
    321 	}
    322 }
    323 
    324 function u32
    325 beamformer_shared_memory_release_pages(u8 *base, u32 first_free_page, u32 page, u32 count, u32 page_size)
    326 {
    327 	u32 current = first_free_page;
    328 	BeamformerFreePageRun *pr = (void *)(base + page_size * current);
    329 	while (current < page && page > pr->next_page) {
    330 		current = pr->next_page;
    331 		pr = (void *)(base + page_size * current);
    332 	}
    333 
    334 	u32 result = first_free_page;
    335 
    336 	BeamformerFreePageRun *to_free = (void *)(base + page_size * page);
    337 	zero_struct(to_free);
    338 
    339 	if (current > page) {
    340 		assert(current == first_free_page);
    341 		assert(pr->prev_page == -1u);
    342 
    343 		result = page;
    344 
    345 		to_free->page_count = count;
    346 		if (page + count == current) {
    347 			to_free->page_count += pr->page_count;
    348 			to_free->next_page   = pr->next_page;
    349 		} else {
    350 			to_free->next_page = current;
    351 			pr->prev_page      = page;
    352 		}
    353 	} else {
    354 		BeamformerFreePageRun *next = (void *)(base + page_size * pr->next_page);
    355 		if (current + pr->page_count == page) {
    356 			pr->page_count += count;
    357 			if (current + pr->page_count == pr->next_page) {
    358 				pr->next_page   = next->next_page;
    359 				pr->page_count += next->page_count;
    360 				if (next->next_page != -1u) {
    361 					next = (void *)(base + page_size * next->next_page);
    362 					next->prev_page = current;
    363 				}
    364 			}
    365 		} else {
    366 			if (pr->next_page != -1u) next->prev_page = page;
    367 			to_free->next_page  = pr->next_page;
    368 			to_free->page_count = count;
    369 			to_free->prev_page  = current;
    370 		}
    371 	}
    372 	return result;
    373 }
    374 
    375 function void *
    376 beamformer_shared_memory_make_handle(BeamformerSharedMemoryHandleKind kind, u64 payload)
    377 {
    378 	static_assert(BeamformerSharedMemoryHandleKind_Count <= U16_MAX, "");
    379 	u64 result = ((u64)kind << 48ul) | (payload & 0x0000fffffffffffful);
    380 	return (void *)result;
    381 }
    382 
    383 function BeamformerSharedMemoryHandleKind
    384 beamformer_shared_memory_handle_kind(void *handle)
    385 {
    386 	BeamformerSharedMemoryHandleKind result = (BeamformerSharedMemoryHandleKind)((u64)handle >> 48ul);
    387 	return result;
    388 }
    389 
    390 function u64
    391 beamformer_shared_memory_handle_payload(void *handle)
    392 {
    393 	u64 result = ((u64)handle & 0x0000fffffffffffful);
    394 	return result;
    395 }
    396 
    397 #define beamformer_shared_memory_transient_alloc(sm, ...) beamformer_shared_memory_transient_alloc_(sm, (ArenaAllocateInfo){.align = 8, .count = 1, __VA_ARGS__})
    398 function BeamformerSharedMemoryAllocationHandle
    399 beamformer_shared_memory_transient_alloc_(BeamformerSharedMemory *sm, ArenaAllocateInfo info)
    400 {
    401 	u64 expected = 0;
    402 	spin_while_false(atomic_cas_u64(&sm->transient.lock, &expected, 1));
    403 
    404 	u64 needed_size = AlignUpPowerOfTwo(info.size * info.count + sizeof(BeamformerSharedMemoryAllocation), info.align);
    405 	u64 page_count  = DivRoundUp(needed_size, BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    406 
    407 	u8 *base = (u8 *)sm + sm->transient.offset;
    408 	BeamformerFreePageRun *pr = sm->transient.first_free_page != -1u ? (void *)(base + sm->transient.first_free_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE) : 0;
    409 	while (pr) {
    410 		if (pr->page_count >= page_count)
    411 			break;
    412 		pr = pr->next_page != -1u ? (void *)(base + pr->next_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE) : 0;
    413 	}
    414 
    415 	BeamformerSharedMemoryAllocationHandle result = 0;
    416 	if (pr) {
    417 		u64 new_free_page = (u64)((u8 *)pr - base) / BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE + page_count;
    418 
    419 		BeamformerFreePageRun *next = (void *)(base + new_free_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    420 		next->page_count = pr->page_count - page_count;
    421 		next->prev_page  = pr->prev_page;
    422 		next->next_page  = pr->next_page;
    423 
    424 		if (next->prev_page != -1u) {
    425 			BeamformerFreePageRun *prev = (void *)(base + next->prev_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    426 			prev->next_page = new_free_page;
    427 		}
    428 
    429 		if (next->next_page != -1u) {
    430 			BeamformerFreePageRun *nn = (void *)(base + next->next_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    431 			nn->prev_page = new_free_page;
    432 		}
    433 
    434 		zero_struct(pr);
    435 
    436 		BeamformerSharedMemoryAllocation *ma = (void *)pr;
    437 		ma->size       = info.size * info.count;
    438 		ma->align      = info.align;
    439 		ma->page_count = page_count;
    440 
    441 		sm->transient.first_free_page = new_free_page;
    442 
    443 		result = beamformer_shared_memory_make_handle(BeamformerSharedMemoryHandleKind_Memory,
    444 		                                              new_free_page - page_count);
    445 	}
    446 	atomic_store_u64(&sm->transient.lock, 0);
    447 
    448 	return result;
    449 }
    450 
    451 function void
    452 beamformer_shared_memory_allocation_release(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle)
    453 {
    454 	assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory);
    455 
    456 	u64 expected = 0;
    457 	spin_while_false(atomic_cas_u64(&sm->transient.lock, &expected, 1));
    458 	{
    459 		u64 page = beamformer_shared_memory_handle_payload(handle);
    460 		u8 *base = (u8 *)sm + sm->transient.offset;
    461 		BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    462 		u32 next_free = beamformer_shared_memory_release_pages(base, sm->transient.first_free_page,
    463 		                                                       page, ma->page_count,
    464 		                                                       BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    465 		sm->transient.first_free_page = next_free;
    466 	}
    467 	atomic_store_u64(&sm->transient.lock, 0);
    468 }
    469 
    470 function void *
    471 beamformer_shared_memory_allocation_pointer(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle)
    472 {
    473 	assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory);
    474 	u64 page = beamformer_shared_memory_handle_payload(handle);
    475 	u8 *base = (u8 *)sm + sm->transient.offset;
    476 	BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    477 	void *result = (void *)AlignUpPowerOfTwo((u64)(ma + 1), (u64)ma->align);
    478 	return result;
    479 }
    480 
    481 function u64
    482 beamformer_shared_memory_allocation_size(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle)
    483 {
    484 	assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory);
    485 	u64 page = beamformer_shared_memory_handle_payload(handle);
    486 
    487 	u8 *base = (u8 *)sm + sm->transient.offset;
    488 	BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
    489 	u64 result = ma->size;
    490 	return result;
    491 }