Commit: 4230dc0ccdf5599738e2d8dff4d69ac4c32fa53d
Parent: ba6fdc1589d2ee5762805d4cc426b58083ae7c96
Author: Randy Palamar
Date: Mon, 26 Oct 2026 19:22:58 -0700
core: shared_memory: introduce transient page allocator
In order to implement some of the ideas we have had for the
library we need to be able to make better use of the transient
area of the shared memory region.
Since both the compute timeline and upload timeline want access to
it we cannot really treat it as a simple arena. Allocations used
for upload have a different lifetime than allocations used for
parameter set interning on the compute timeline yet their order of
both usage and allocation is not fixed.
This can be solved by using some slightly more general allocation
strategy like a slab/page allocator. While in theory this adds
more overhead I was not able to measure a reduction in maximum
throughput achievable with this method (actually it improved a
minor amount due to moving locks around). That's not to say we are
achieving maximum throughput. It is still limited to ~19GB/s on my
system which should be able to achieve ~72GB/s to RAM or
(limiting) 63GB/s across PCIe. But thats a topic for another time.
Diffstat:
7 files changed, 354 insertions(+), 168 deletions(-)
diff --git a/beamformer.c b/beamformer.c
@@ -234,29 +234,42 @@ beamformer_init(BeamformerInput *input)
load_gl(&ctx->error_stream);
- ctx->shared_memory = input->shared_memory;
- ctx->shared_memory_size = input->shared_memory_size;
- if (ctx->shared_memory_size < (i64)sizeof(*ctx->shared_memory))
- fatal(str8("Get more ram lol\n"));
- zero_struct(ctx->shared_memory);
-
- ctx->shared_memory->version = BEAMFORMER_SHARED_MEMORY_VERSION;
- ctx->shared_memory->reserved_parameter_blocks = 1;
+ ////////////////////////////
+ // NOTE(rnp): shared memory setup
{
- BeamformerParameterBlock *pb = beamformer_parameter_block(ctx->shared_memory, 0);
- zero_struct(pb);
- }
+ BeamformerSharedMemory *sm = input->shared_memory;
+ ctx->shared_memory = sm;
+ ctx->shared_memory_size = input->shared_memory_size;
+ if (ctx->shared_memory_size < (i64)sizeof(*ctx->shared_memory))
+ fatal(str8("Get more ram lol\n"));
+ zero_struct(sm);
+
+ sm->version = BEAMFORMER_SHARED_MEMORY_VERSION;
+ sm->reserved_parameter_blocks = 1;
+ {
+ BeamformerParameterBlock *pb = beamformer_parameter_block(sm, 0);
+ zero_struct(pb);
+ }
- ctx->shared_memory->beamformed_frame_buffer_size = cs->backlog.buffer->size;
+ sm->beamformed_frame_buffer_size = cs->backlog.buffer->size;
- // TODO(rnp): dynamic rf data buffer slot usage
- // NOTE(rnp): will be same as the max size we were able to get for the frame buffer
- ctx->shared_memory->capabilities.max_rf_data_size = cs->backlog.buffer->size
- / BeamformerMaxRawDataFramesInFlight;
+ // TODO(rnp): dynamic rf data buffer slot usage
+ // NOTE(rnp): will be same as the max size we were able to get for the frame buffer
+ sm->capabilities.max_rf_data_size = cs->backlog.buffer->size / BeamformerMaxRawDataFramesInFlight;
- ctx->shared_memory->capabilities.cuda = cuda_supported();
- // TODO(rnp): re-enable hilbert support, with and without cuda
- ctx->shared_memory->capabilities.hilbert = 0;
+ sm->capabilities.cuda = cuda_supported();
+ // TODO(rnp): re-enable hilbert support, with and without cuda
+ sm->capabilities.hilbert = 0;
+
+ sm->transient.offset = AlignUpPowerOfTwo(sizeof(*sm) + BeamformerMaxParameterBlocks * sizeof(BeamformerParameterBlock),
+ BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ sm->transient.page_count = DivRoundUp(ctx->shared_memory_size - sm->transient.offset,
+ BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ sm->transient.first_free_page = 0;
+ BeamformerFreePageRun *pr = (void *)((u8 *)sm + sm->transient.offset);
+ pr->page_count = sm->transient.page_count;
+ pr->next_page = pr->prev_page = -1u;
+ }
/* TODO(rnp): I'm not sure if its a good idea to pre-reserve a bunch of semaphores
* on w32 but thats what we are doing for now */
diff --git a/beamformer_core.c b/beamformer_core.c
@@ -1433,14 +1433,16 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
case BeamformerWorkKind_ExportBuffer:{
/* TODO(rnp): better way of handling DispatchCompute barrier */
post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute);
- beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)work->lock, (u32)-1);
BeamformerExportContext *ec = &work->export_context;
switch (ec->kind) {
case BeamformerExportKind_BeamformedData:{
BeamformerFrameBacklog *bl = &ctx->compute_context.backlog;
u32 req_count = Clamp(ec->count, 1, bl->counter);
u32 frame_idx = bl->counter - req_count;
- u8 *sm_output = beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size);
+
+ u8 *sm_output = beamformer_shared_memory_allocation_pointer(sm, ec->memory);
+ u64 total_size = beamformer_shared_memory_allocation_size(sm, ec->memory);
+
u64 exported_size = 0;
for (u32 export_count = 0; export_count < req_count; export_count++, frame_idx++) {
BeamformerFrame *f = bl->frames + frame_idx % countof(bl->frames);
@@ -1449,7 +1451,7 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
// NOTE(tkh) we don't want to assume that all req_count frames are the same size,
// so we either need to count the total size of all requested frames first or
// just fill up as much as possible.
- if (exported_size + frame_size <= ec->size) {
+ if (exported_size + frame_size <= total_size) {
u64 offset = f->gpu_pointer - bl->buffer->gpu_pointer;
gpu_host_wait_timeline(GPUTimeline_Compute, f->timeline_valid_value, -1ULL);
gpu_buffer_range_download(sm_output + exported_size, bl->buffer, offset, frame_size, 1);
@@ -1463,13 +1465,13 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
/* NOTE(rnp): do a little spin to let this finish updating */
spin_while_true(table->write_index != atomic_load_u32(&table->read_index));
ComputeShaderStats *stats = ctx->compute_shader_stats;
- if (sizeof(stats->table) <= ec->size)
- memory_copy(beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size),
- &stats->table, sizeof(stats->table));
+
+ u64 size = beamformer_shared_memory_allocation_size(sm, ec->memory);
+ if (sizeof(stats->table) <= size)
+ memory_copy(beamformer_shared_memory_allocation_pointer(sm, ec->memory), &stats->table, sizeof(stats->table));
}break;
InvalidDefaultCase;
}
- beamformer_shared_memory_release_lock(ctx->shared_memory, work->lock);
post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync);
}break;
@@ -1784,19 +1786,17 @@ DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute)
DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
{
- BeamformerSharedMemory *sm = ctx->shared_memory;
- BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace;
- BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF;
+ BeamformerSharedMemory *sm = ctx->shared_memory;
+ BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF;
- u64 rf_block_rf_size;
- if (atomic_load_u32(sm->locks + upload_lock) &&
- (rf_block_rf_size = atomic_swap_u64(&sm->rf_block_rf_size, 0)))
+ BeamformerSharedMemoryAllocationHandle rf_handle;
+ if (atomic_load_u32(sm->locks + upload_lock) && (rf_handle = atomic_swap_u64(&sm->rf_handle, 0)))
{
- beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)scratch_lock, (u32)-1);
-
BeamformerRFBuffer *rf = ctx->rf_buffer;
- rf->active_rf_size = gpu_round_up_to_sync_size(rf_block_rf_size & 0x00FFFFFFFFFFFFFFull, 64);
+ u64 rf_size = beamformer_shared_memory_allocation_size(sm, rf_handle);
+
+ rf->active_rf_size = gpu_round_up_to_sync_size(rf_size, BEAMFORMER_RF_ALIGN);
if unlikely((u64)rf->buffer.size < countof(rf->upload_complete_values) * rf->active_rf_size) {
gpu_buffer_allocate(&rf->buffer, (GPUBufferAllocateInfo){
.size = countof(rf->upload_complete_values) * rf->active_rf_size,
@@ -1814,7 +1814,7 @@ DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
gpu_host_wait_timeline(GPUTimeline_Compute, rf->compute_complete_values[slot], -1ULL);
assert((ctx->shared_memory_size % os_system_info()->page_size) == 0 &&
- (os_system_info()->page_size % gpu_round_up_to_sync_size(1, 64)) == 0);
+ (os_system_info()->page_size % gpu_round_up_to_sync_size(1, BEAMFORMER_RF_ALIGN)) == 0);
u64 wait_value = gpu_semaphore_value(rf->upload_semaphore) + 1;
atomic_store_u64(rf->upload_complete_values + slot, wait_value);
@@ -1822,14 +1822,15 @@ DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
void *host_pointer = gpu_buffer_host_pointer(&rf->buffer);
memory_copy_non_temporal((u8 *)host_pointer + slot * rf->active_rf_size,
- beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size),
+ beamformer_shared_memory_allocation_pointer(sm, rf_handle),
rf->active_rf_size);
store_fence();
GPUSemaphoreSignalInfo signal_info = {.semaphore = rf->upload_semaphore, .value = wait_value};
gpu_buffer_make_visible(&rf->buffer, slot * rf->active_rf_size, rf->active_rf_size, &signal_info, 1);
- beamformer_shared_memory_release_lock(ctx->shared_memory, (i32)scratch_lock);
+ beamformer_shared_memory_allocation_release(sm, rf_handle);
+
post_sync_barrier(ctx->shared_memory, upload_lock);
os_wake_all_waiters(ctx->compute_worker_sync);
diff --git a/beamformer_shared_memory.c b/beamformer_shared_memory.c
@@ -1,5 +1,31 @@
/* See LICENSE for license details. */
-#define BEAMFORMER_SHARED_MEMORY_VERSION (37UL)
+#define BEAMFORMER_SHARED_MEMORY_VERSION (38UL)
+
+#define BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE KB(32)
+
+#define BEAMFORMER_RF_ALIGN 256
+
+#define BEAMFORMER_HANDLE(kind) struct kind##_T; typedef struct kind##_T *kind
+
+BEAMFORMER_HANDLE(BeamformerSharedMemoryAllocationHandle);
+
+typedef enum {
+ BeamformerSharedMemoryHandleKind_Nil,
+ BeamformerSharedMemoryHandleKind_Memory,
+ BeamformerSharedMemoryHandleKind_Count,
+} BeamformerSharedMemoryHandleKind;
+
+typedef struct {
+ u64 size;
+ u32 align;
+ u32 page_count;
+} BeamformerSharedMemoryAllocation;
+
+typedef struct {
+ u32 page_count;
+ u32 prev_page;
+ u32 next_page;
+} BeamformerFreePageRun;
typedef enum {
BeamformerWorkKind_Compute,
@@ -23,12 +49,11 @@ typedef enum {
typedef struct {
BeamformerExportKind kind;
- u32 count; /* Number of items to export */
- u64 size; /* Total expected size of the exported data */
+ u32 count; /* Number of frames to export */
+ BeamformerSharedMemoryAllocationHandle memory;
} BeamformerExportContext;
#define BEAMFORMER_SHARED_MEMORY_LOCKS \
- X(ScratchSpace) \
X(UploadRF) \
X(ExportSync) \
X(DispatchCompute)
@@ -142,9 +167,7 @@ typedef struct {
u32 reserved_parameter_blocks;
/* TODO(rnp): this is really sucky. we need a better way to communicate this */
- u64 rf_block_rf_size;
-
- b64 new_rf_upload;
+ BeamformerSharedMemoryAllocationHandle rf_handle;
// NOTE(rnp): currently this cannot be directly user readable. its interpretation
// requires beamformer implementation details
@@ -159,6 +182,13 @@ typedef struct {
BeamformerLiveImagingParameters live_imaging_parameters;
BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags;
+ struct {
+ u32 first_free_page;
+ u32 page_count;
+ u64 offset;
+ u64 lock;
+ } transient;
+
BeamformWorkQueue external_work_queue;
} BeamformerSharedMemory;
@@ -274,25 +304,6 @@ beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block)
beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block);
}
-function Arena *
-beamformer_shared_memory_scratch_arena(BeamformerSharedMemory *sm, i64 shared_memory_size)
-{
- assert(sm->reserved_parameter_blocks > 0);
- BeamformerParameterBlock *last = beamformer_parameter_block(sm, sm->reserved_parameter_blocks - 1);
- u64 size = (u64)((u8 *)sm + shared_memory_size - (u8 *)(last + 1));
- Arena *result = arena_create(.optional_backing_store = (last + 1), .reserve_size = size, .commit_size = size, .flags = ArenaFlag_NoChain);
- arena_pre_align(result, KB(4));
- return result;
-}
-
-function void *
-beamformer_shared_memory_data_pointer(BeamformerSharedMemory *sm, i64 shared_memory_size)
-{
- Arena *arena = beamformer_shared_memory_scratch_arena(sm, shared_memory_size);
- void *result = (u8 *)arena + arena->position;
- return result;
-}
-
function void
beamformed_shared_memory_dirty_flag(BeamformerSharedMemory *sm, u32 block, BeamformerParameterDirtyFlags flag)
{
@@ -309,3 +320,172 @@ post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind loc
beamformer_shared_memory_release_lock(sm, lock);
}
}
+
+function u32
+beamformer_shared_memory_release_pages(u8 *base, u32 first_free_page, u32 page, u32 count, u32 page_size)
+{
+ u32 current = first_free_page;
+ BeamformerFreePageRun *pr = (void *)(base + page_size * current);
+ while (current < page && page > pr->next_page) {
+ current = pr->next_page;
+ pr = (void *)(base + page_size * current);
+ }
+
+ u32 result = first_free_page;
+
+ BeamformerFreePageRun *to_free = (void *)(base + page_size * page);
+ zero_struct(to_free);
+
+ if (current > page) {
+ assert(current == first_free_page);
+ assert(pr->prev_page == -1u);
+
+ result = page;
+
+ to_free->page_count = count;
+ if (page + count == current) {
+ to_free->page_count += pr->page_count;
+ to_free->next_page = pr->next_page;
+ } else {
+ to_free->next_page = current;
+ pr->prev_page = page;
+ }
+ } else {
+ BeamformerFreePageRun *next = (void *)(base + page_size * pr->next_page);
+ if (current + pr->page_count == page) {
+ pr->page_count += count;
+ if (current + pr->page_count == pr->next_page) {
+ pr->next_page = next->next_page;
+ pr->page_count += next->page_count;
+ if (next->next_page != -1u) {
+ next = (void *)(base + page_size * next->next_page);
+ next->prev_page = current;
+ }
+ }
+ } else {
+ if (pr->next_page != -1u) next->prev_page = page;
+ to_free->next_page = pr->next_page;
+ to_free->page_count = count;
+ to_free->prev_page = current;
+ }
+ }
+ return result;
+}
+
+function void *
+beamformer_shared_memory_make_handle(BeamformerSharedMemoryHandleKind kind, u64 payload)
+{
+ static_assert(BeamformerSharedMemoryHandleKind_Count <= U16_MAX, "");
+ u64 result = ((u64)kind << 48ul) | (payload & 0x0000fffffffffffful);
+ return (void *)result;
+}
+
+function BeamformerSharedMemoryHandleKind
+beamformer_shared_memory_handle_kind(void *handle)
+{
+ BeamformerSharedMemoryHandleKind result = (BeamformerSharedMemoryHandleKind)((u64)handle >> 48ul);
+ return result;
+}
+
+function u64
+beamformer_shared_memory_handle_payload(void *handle)
+{
+ u64 result = ((u64)handle & 0x0000fffffffffffful);
+ return result;
+}
+
+#define beamformer_shared_memory_transient_alloc(sm, ...) beamformer_shared_memory_transient_alloc_(sm, (ArenaAllocateInfo){.align = 8, .count = 1, __VA_ARGS__})
+function BeamformerSharedMemoryAllocationHandle
+beamformer_shared_memory_transient_alloc_(BeamformerSharedMemory *sm, ArenaAllocateInfo info)
+{
+ u64 expected = 0;
+ spin_while_false(atomic_cas_u64(&sm->transient.lock, &expected, 1));
+
+ u64 needed_size = AlignUpPowerOfTwo(info.size * info.count + sizeof(BeamformerSharedMemoryAllocation), info.align);
+ u64 page_count = DivRoundUp(needed_size, BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+
+ u8 *base = (u8 *)sm + sm->transient.offset;
+ BeamformerFreePageRun *pr = sm->transient.first_free_page != -1u ? (void *)(base + sm->transient.first_free_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE) : 0;
+ while (pr) {
+ if (pr->page_count >= page_count)
+ break;
+ pr = pr->next_page != -1u ? (void *)(base + pr->next_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE) : 0;
+ }
+
+ BeamformerSharedMemoryAllocationHandle result = 0;
+ if (pr) {
+ u64 new_free_page = (u64)((u8 *)pr - base) / BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE + page_count;
+
+ BeamformerFreePageRun *next = (void *)(base + new_free_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ next->page_count = pr->page_count - page_count;
+ next->prev_page = pr->prev_page;
+ next->next_page = pr->next_page;
+
+ if (next->prev_page != -1u) {
+ BeamformerFreePageRun *prev = (void *)(base + next->prev_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ prev->next_page = new_free_page;
+ }
+
+ if (next->next_page != -1u) {
+ BeamformerFreePageRun *nn = (void *)(base + next->next_page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ nn->prev_page = new_free_page;
+ }
+
+ zero_struct(pr);
+
+ BeamformerSharedMemoryAllocation *ma = (void *)pr;
+ ma->size = info.size * info.count;
+ ma->align = info.align;
+ ma->page_count = page_count;
+
+ sm->transient.first_free_page = new_free_page;
+
+ result = beamformer_shared_memory_make_handle(BeamformerSharedMemoryHandleKind_Memory,
+ new_free_page - page_count);
+ }
+ atomic_store_u64(&sm->transient.lock, 0);
+
+ return result;
+}
+
+function void
+beamformer_shared_memory_allocation_release(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle)
+{
+ assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory);
+
+ u64 expected = 0;
+ spin_while_false(atomic_cas_u64(&sm->transient.lock, &expected, 1));
+ {
+ u64 page = beamformer_shared_memory_handle_payload(handle);
+ u8 *base = (u8 *)sm + sm->transient.offset;
+ BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ u32 next_free = beamformer_shared_memory_release_pages(base, sm->transient.first_free_page,
+ page, ma->page_count,
+ BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ sm->transient.first_free_page = next_free;
+ }
+ atomic_store_u64(&sm->transient.lock, 0);
+}
+
+function void *
+beamformer_shared_memory_allocation_pointer(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle)
+{
+ assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory);
+ u64 page = beamformer_shared_memory_handle_payload(handle);
+ u8 *base = (u8 *)sm + sm->transient.offset;
+ BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ void *result = (void *)AlignUpPowerOfTwo((u64)(ma + 1), (u64)ma->align);
+ return result;
+}
+
+function u64
+beamformer_shared_memory_allocation_size(BeamformerSharedMemory *sm, BeamformerSharedMemoryAllocationHandle handle)
+{
+ assert(beamformer_shared_memory_handle_kind(handle) == BeamformerSharedMemoryHandleKind_Memory);
+ u64 page = beamformer_shared_memory_handle_payload(handle);
+
+ u8 *base = (u8 *)sm + sm->transient.offset;
+ BeamformerSharedMemoryAllocation *ma = (void *)(base + page * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE);
+ u64 result = ma->size;
+ return result;
+}
diff --git a/lib/ogl_beamformer_lib.c b/lib/ogl_beamformer_lib.c
@@ -27,9 +27,9 @@ W32(iptr) OpenFileMappingA(u32, b32, c8 *);
global struct {
BeamformerSharedMemory *bp;
+ b32 new_rf;
i32 timeout_ms;
BeamformerLibErrorKind last_error;
- i64 shared_memory_size;
} g_beamformer_library_context;
#if OS_LINUX
@@ -156,8 +156,7 @@ check_shared_memory(void)
BeamformerSharedMemory *bp = (BeamformerSharedMemory *)shared_memory.data;
result = lib_error_check(bp->version == BEAMFORMER_SHARED_MEMORY_VERSION, VersionMismatch);
if (result) {
- g_beamformer_library_context.bp = bp;
- g_beamformer_library_context.shared_memory_size = shared_memory.length;
+ g_beamformer_library_context.bp = bp;
} else {
os_close_shared_memory_area(shared_memory.data, shared_memory.length);
}
@@ -314,9 +313,9 @@ beamformer_maximum_rf_data_size(void)
{
u64 result = U64_MAX;
if (check_shared_memory()) {
- Arena *sm = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp,
- g_beamformer_library_context.shared_memory_size);
- result = Min(sm->reserved - sm->position, g_beamformer_library_context.bp->capabilities.max_rf_data_size);
+ u64 transient_size = g_beamformer_library_context.bp->transient.page_count
+ * BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE;
+ result = Min(transient_size, g_beamformer_library_context.bp->capabilities.max_rf_data_size);
}
return result;
}
@@ -487,77 +486,79 @@ function b32
beamformer_push_data_base(void *data, u64 data_size, i32 timeout_ms, u32 block)
{
b32 result = 0;
- Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp,
- g_beamformer_library_context.shared_memory_size);
- BeamformerParameterBlock *b = beamformer_parameter_block(g_beamformer_library_context.bp, block);
+ BeamformerSharedMemory *sm = g_beamformer_library_context.bp;
+ BeamformerParameterBlock *b = beamformer_parameter_block(sm, block);
BeamformerParameters *bp = &b->parameters;
BeamformerDataKind data_kind = b->pipeline.data_kind;
BeamformerContrastMode contrast_mode = bp->contrast_mode;
-
- u64 arena_size = scratch->reserved - scratch->position;
- u64 max_rf_size = g_beamformer_library_context.bp->capabilities.max_rf_data_size;
+ u64 max_rf_size = sm->capabilities.max_rf_data_size;
u32 rf_size = bp->acquisition_count * bp->sample_count * bp->channel_count * beamformer_data_kind_byte_size[data_kind];
u32 raw_size = bp->raw_data_dimensions.x * bp->raw_data_dimensions.y * beamformer_data_kind_byte_size[data_kind];
// TODO(rnp): support multi push upload so that max_rf_size is actual limit
- if (lib_error_check(rf_size <= arena_size, BufferOverflow) &&
- lib_error_check(rf_size <= max_rf_size, RFDataSizeOverflow) &&
- lib_error_check(rf_size <= data_size && data_size == raw_size, DataSizeMismatch))
- {
- if (lib_try_lock(BeamformerSharedMemoryLockKind_UploadRF, timeout_ms)) {
- if (lib_try_lock(BeamformerSharedMemoryLockKind_ScratchSpace, 0)) {
- u32 channel_count = bp->channel_count;
- u32 out_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->sample_count * bp->acquisition_count;
- u32 in_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->raw_data_dimensions.x;
-
- for (u32 channel = 0; channel < channel_count; channel++) {
- u16 data_channel = (u16)b->channel_mapping[channel];
- u32 out_off = out_channel_stride * channel;
- u32 in_off = in_channel_stride * data_channel;
- u8 *memory = (u8 *)scratch + scratch->position + out_off;
- switch (contrast_mode) {
- default:{
- /* NOTE(rnp): non temporal copy would be better, but we can't ensure
- * 64 byte boundaries. */
- memory_copy(memory, (u8 *)data + in_off, out_channel_stride);
- }break;
-
- case BeamformerContrastMode_A1S2:{
- read_only u8 reduce_a1s2_index_map[] = {
- [BeamformerDataKind_Int16] = 0,
- [BeamformerDataKind_Int16Complex] = 0,
- [BeamformerDataKind_Float32] = 1,
- [BeamformerDataKind_Float32Complex] = 1,
- [BeamformerDataKind_Float16] = 2,
- [BeamformerDataKind_Float16Complex] = 2,
- };
- static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), "");
-
- local_persist beamformer_reduce_a1s2_contrast_fn *reduce_a1s2_fn_table[] = {
- #define X(type, ...) beamformer_reduce_a1s2_contrast_##type,
- BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST
- #undef X
- };
-
- u32 sample_count = bp->sample_count * beamformer_data_kind_element_count[data_kind];
- u32 acquisition_stride = sample_count * beamformer_data_kind_element_size[data_kind];
- for (u32 acquisition = 0; acquisition < bp->acquisition_count; acquisition++) {
- void *out = memory + acquisition * acquisition_stride;
- void *in = (u8 *)data + in_off + 3 * acquisition * acquisition_stride;
- reduce_a1s2_fn_table[reduce_a1s2_index_map[data_kind]](out, in, sample_count);
- }
- }break;
+ if (lib_try_lock(BeamformerSharedMemoryLockKind_UploadRF, timeout_ms)) {
+ BeamformerSharedMemoryAllocationHandle rf_handle = beamformer_shared_memory_transient_alloc(sm, .size = rf_size, .align = BEAMFORMER_RF_ALIGN);
+ if (lib_error_check(rf_handle != 0, TransientSpaceOverflow) &&
+ lib_error_check(rf_size <= max_rf_size, RFDataSizeOverflow) &&
+ lib_error_check(rf_size <= data_size && data_size == raw_size, DataSizeMismatch))
+ {
+ u32 channel_count = bp->channel_count;
+ u32 out_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->sample_count * bp->acquisition_count;
+ u32 in_channel_stride = beamformer_data_kind_byte_size[data_kind] * bp->raw_data_dimensions.x;
+
+ u8 *base = beamformer_shared_memory_allocation_pointer(sm, rf_handle);
+ for (u32 channel = 0; channel < channel_count; channel++) {
+ u16 data_channel = (u16)b->channel_mapping[channel];
+ u32 out_off = out_channel_stride * channel;
+ u32 in_off = in_channel_stride * data_channel;
+
+ u8 *memory = base + out_off;
+ switch (contrast_mode) {
+ default:{
+ /* NOTE(rnp): non temporal copy would be better, but we can't ensure
+ * 64 byte boundaries. */
+ memory_copy(memory, (u8 *)data + in_off, out_channel_stride);
+ }break;
+
+ case BeamformerContrastMode_A1S2:{
+ read_only u8 reduce_a1s2_index_map[] = {
+ [BeamformerDataKind_Int16] = 0,
+ [BeamformerDataKind_Int16Complex] = 0,
+ [BeamformerDataKind_Float32] = 1,
+ [BeamformerDataKind_Float32Complex] = 1,
+ [BeamformerDataKind_Float16] = 2,
+ [BeamformerDataKind_Float16Complex] = 2,
+ };
+ static_assert(BeamformerDataKind_Float16Complex == (BeamformerDataKind_Count - 1), "");
+
+ local_persist beamformer_reduce_a1s2_contrast_fn *reduce_a1s2_fn_table[] = {
+ #define X(type, ...) beamformer_reduce_a1s2_contrast_##type,
+ BEAMFORMER_REDUCE_A1S2_CONTRAST_LIST
+ #undef X
+ };
+ beamformer_reduce_a1s2_contrast_fn *reduce = reduce_a1s2_fn_table[reduce_a1s2_index_map[data_kind]];
+
+ u32 sample_count = bp->sample_count * beamformer_data_kind_element_count[data_kind];
+ u32 acquisition_stride = sample_count * beamformer_data_kind_element_size[data_kind];
+ for (u32 acquisition = 0; acquisition < bp->acquisition_count; acquisition++) {
+ void *out = memory + acquisition * acquisition_stride;
+ void *in = (u8 *)data + in_off + 3 * acquisition * acquisition_stride;
+ reduce(out, in, sample_count);
}
+ }break;
}
-
- lib_release_lock(BeamformerSharedMemoryLockKind_ScratchSpace);
- /* TODO(rnp): need a better way to communicate this */
- u64 rf_block_rf_size = ((u64)block << 56ull) | (rf_size & 0x00FFFFFFFFFFFFFFull);
- atomic_store_u64(&g_beamformer_library_context.bp->rf_block_rf_size, rf_block_rf_size);
- atomic_store_u64(&g_beamformer_library_context.bp->new_rf_upload, 1ull);
- result = 1;
}
+
+ /* TODO(rnp): need a better way to communicate this */
+ atomic_store_u64(&sm->rf_handle, rf_handle);
+ g_beamformer_library_context.new_rf = 1;
+ result = 1;
+ }
+
+ if (!result && rf_handle) {
+ beamformer_shared_memory_allocation_release(sm, rf_handle);
+ lib_release_lock(BeamformerSharedMemoryLockKind_UploadRF);
}
}
return result;
@@ -578,7 +579,8 @@ beamformer_start_compute(u64 offset, u32 image_plane_tag, u32 parameter_slot)
BeamformWork *work = try_push_work_queue();
if (work) {
- b64 new_rf = atomic_swap_u64(&g_beamformer_library_context.bp->new_rf_upload, 0ull);
+ b32 new_rf = g_beamformer_library_context.new_rf;
+ g_beamformer_library_context.new_rf = 0;
work->kind = new_rf ? BeamformerWorkKind_WaitThenCompute : BeamformerWorkKind_Compute;
work->compute_context.view_plane = image_plane_tag;
work->compute_context.parameter_set = parameter_slot;
@@ -683,7 +685,6 @@ beamformer_export_buffer(BeamformerExportContext export_context)
if (result) {
work->export_context = export_context;
work->kind = BeamformerWorkKind_ExportBuffer;
- work->lock = BeamformerSharedMemoryLockKind_ScratchSpace;
beamform_work_queue_push_commit(&g_beamformer_library_context.bp->external_work_queue);
}
return result;
@@ -700,14 +701,11 @@ beamformer_export(BeamformerExportContext export, void *out, i32 timeout_ms)
beamformer_flush_commands();
if (lib_try_lock(BeamformerSharedMemoryLockKind_ExportSync, timeout_ms)) {
- if (lib_try_lock(BeamformerSharedMemoryLockKind_ScratchSpace, 0)) {
- void *sm = beamformer_shared_memory_data_pointer(g_beamformer_library_context.bp,
- g_beamformer_library_context.shared_memory_size);
- memory_copy(out, sm, export.size);
- lib_release_lock(BeamformerSharedMemoryLockKind_ScratchSpace);
- result = 1;
- }
+ BeamformerSharedMemory *sm = g_beamformer_library_context.bp;
+ memory_copy(out, beamformer_shared_memory_allocation_pointer(sm, export.memory),
+ beamformer_shared_memory_allocation_size(sm, export.memory));
lib_release_lock(BeamformerSharedMemoryLockKind_ExportSync);
+ result = 1;
}
}
return result;
@@ -719,8 +717,17 @@ beamformer_get_last_frames(void *out_data, u64 out_data_size, u32 count)
BeamformerExportContext export = {0};
export.kind = BeamformerExportKind_BeamformedData;
export.count = count;
- export.size = out_data_size;
- b32 result = out_data && out_data_size && count && beamformer_export(export, out_data, g_beamformer_library_context.timeout_ms);
+ if (out_data_size) {
+ export.memory = beamformer_shared_memory_transient_alloc(g_beamformer_library_context.bp,
+ .size = out_data_size, .align = 64);
+ }
+
+ b32 result = lib_error_check(export.memory != 0, TransientSpaceOverflow) && count;
+ if (result)
+ result = beamformer_export(export, out_data, g_beamformer_library_context.timeout_ms);
+ if (export.memory)
+ beamformer_shared_memory_allocation_release(g_beamformer_library_context.bp, export.memory);
+
return result;
}
@@ -746,16 +753,10 @@ beamformer_beamform_data(BeamformerSimpleParameters *bp, void *data, uint32_t da
u64 output_size = output_points.x * output_points.y * output_points.z * sizeof(f32);
if (complex) output_size *= 2;
- Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp,
- g_beamformer_library_context.shared_memory_size);
- u64 scratch_size = scratch->reserved - scratch->position;
- if (result && out_data) result &= lib_error_check(output_size <= scratch_size, ExportSpaceOverflow);
+ result = beamformer_push_data_with_compute(data, data_size, 0, 0);
- if (result) {
- result = beamformer_push_data_with_compute(data, data_size, 0, 0);
- if (result && out_data)
- result = beamformer_get_last_frames(out_data, output_size, 1);
- }
+ if (result && out_data)
+ result = beamformer_get_last_frames(out_data, output_size, 1);
}
return result;
}
@@ -765,14 +766,14 @@ beamformer_compute_timings(BeamformerComputeStatsTable *output, i32 timeout_ms)
{
b32 result = 0;
if (check_shared_memory()) {
- Arena *scratch = beamformer_shared_memory_scratch_arena(g_beamformer_library_context.bp,
- g_beamformer_library_context.shared_memory_size);
- u64 scratch_size = scratch->reserved - scratch->position;
- if (lib_error_check(sizeof(*output) <= scratch_size, ExportSpaceOverflow)) {
- BeamformerExportContext export = {0};
- export.kind = BeamformerExportKind_Stats;
- export.size = sizeof(*output);
+ BeamformerExportContext export = {0};
+ export.kind = BeamformerExportKind_Stats;
+ export.memory = beamformer_shared_memory_transient_alloc(g_beamformer_library_context.bp,
+ .size = sizeof(BeamformerComputeStatsTable),
+ .align = alignof(BeamformerComputeStatsTable));
+ if (lib_error_check(export.memory != 0, TransientSpaceOverflow)) {
result = beamformer_export(export, output, timeout_ms);
+ beamformer_shared_memory_allocation_release(g_beamformer_library_context.bp, export.memory);
}
}
return result;
diff --git a/lib/ogl_beamformer_lib_base.h b/lib/ogl_beamformer_lib_base.h
@@ -24,7 +24,7 @@
X(BufferOverflow, 13, "passed buffer size exceeds available space") \
X(DataSizeMismatch, 14, "data size doesn't match the size specified in parameters") \
X(WorkQueueFull, 15, "work queue full") \
- X(ExportSpaceOverflow, 16, "not enough space for data export") \
+ X(TransientSpaceOverflow, 16, "temporary memory space exhausted") \
X(SharedMemory, 17, "failed to open shared memory region") \
X(SyncVariable, 18, "failed to acquire lock within timeout period") \
X(FrameSizeOverflow, 19, "maximum frame size exceeded") \
diff --git a/util.c b/util.c
@@ -284,16 +284,6 @@ arena_clear(Arena *arena)
arena_pop_to(arena, 0);
}
-function void
-arena_pre_align(Arena *arena, u64 align)
-{
- assert(IsPowerOfTwo(align));
- Arena *current = arena->current;
- u8 *start = (u8 *)current + current->position;
- u8 *desired_start = (u8 *)AlignUpPowerOfTwo((u64)start, align);
- current->position += (u64)(desired_start - start);
-}
-
function Temp
temp_begin(Arena *arena)
{
diff --git a/util.h b/util.h
@@ -74,6 +74,7 @@
#define IsPowerOfTwo(a) (((a) & ((a) - 1)) == 0)
#define AlignUpPowerOfTwo(v, a) (((v) + (a) - 1) & (~((a) - 1)))
+#define DivRoundUp(a, b) (((a) + (b) - 1) / (b))
#define IsDigit(c) (Between((c), '0', '9'))
#define IsUpper(c) (((c) & 0x20u) == 0)