Commit: 226ad51e3007ba8efc9af3bfb5cd9bc88b6d95d1
Parent: 7b3b09ecd7264f83adb143e13da624d3c1d103b1
Author: Randy Palamar
Date: Fri, 14 Aug 2026 20:36:22 -0700
core/gpu: add temporary buffer allocation in plan_compute_pipeline
there are a handful temporary GPU results and inputs that are kind
of need to exist outside the beamformed image ring buffer. These
can all be determined during plan_compute_pipeline() and we can
allocate a GPU side arena for containing them.
Diffstat:
11 files changed, 276 insertions(+), 179 deletions(-)
diff --git a/base_types.h b/base_types.h
@@ -119,6 +119,8 @@ typedef union {
f32 E[16];
} m4;
+typedef struct {u8 *start, *end;} BumpArena;
+
typedef enum {
ArenaFlag_NoChain = 1 << 0,
diff --git a/beamformer.c b/beamformer.c
@@ -218,7 +218,7 @@ beamformer_init(BeamformerInput *input)
.timelines_used = timelines,
.label = str8("BeamformedData"),
};
- vk_buffer_allocate(cs->backlog.buffer, &allocate_info);
+ gpu_buffer_allocate(cs->backlog.buffer, &allocate_info);
if (cs->backlog.buffer->size > 0)
break;
}
diff --git a/beamformer.meta b/beamformer.meta
@@ -438,6 +438,7 @@
@Bake
{
[ArrayParameters U64]
+ [IncoherentFrame U64]
[AcquisitionKind U32]
[Sparse B32]
[AcquisitionCount S32]
@@ -458,6 +459,10 @@
[FocusDepth F32]
[TransmitAngle F32]
+ [OutputSizeX U32]
+ [OutputSizeY U32]
+ [OutputSizeZ U32]
+
[ReadiGroupCount U32]
}
@@ -467,11 +472,7 @@
[voxel_transform M4]
[xdc_element_pitch V2]
[output_frame U64]
- [incoherent_frame U64]
[rf_element_offset U32]
- [output_size_x U32]
- [output_size_y U32]
- [output_size_z U32]
[channel_offset S32]
[readi_group U32]
}
@@ -501,12 +502,16 @@
{
@PushConstants
{
- [left_side_buffer U64]
- [right_side_buffer U64]
- [scale F32]
- [output_size_x U32]
- [output_size_y U32]
- [output_size_z U32]
+ [coherent_sum U64]
+ }
+
+ @Bake
+ {
+ [IncoherentSum U64]
+ [Scale F32]
+ [OutputSizeX U32]
+ [OutputSizeY U32]
+ [OutputSizeZ U32]
}
}
diff --git a/beamformer_core.c b/beamformer_core.c
@@ -198,9 +198,9 @@ beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block)
assert(block < countof(cc->compute_plans));
BeamformerComputePlan *cp = cc->compute_plans[block];
if (cp) {
- vk_buffer_release(&cp->array_parameters);
+ gpu_buffer_release(&cp->array_parameters);
for (u32 i = 0; i < countof(cp->filters); i++)
- vk_buffer_release(&cp->filters[i].buffer);
+ gpu_buffer_release(&cp->filters[i].buffer);
cc->compute_plans[block] = 0;
SLLPushFreelist(cp, cc->compute_plan_freelist);
}
@@ -229,7 +229,7 @@ beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena
.flags = VulkanUsageFlag_HostReadWrite,
.label = stream_to_str8(&label),
};
- vk_buffer_allocate(&result->array_parameters, &allocate_info);
+ gpu_buffer_allocate(&result->array_parameters, &allocate_info);
assert((result->array_parameters.gpu_pointer & 63) == 0);
}
return result;
@@ -283,9 +283,9 @@ beamformer_filter_update(BeamformerFilter *f, BeamformerFilterParameters fp, u32
.flags = VulkanUsageFlag_HostReadWrite,
.label = label,
};
- vk_buffer_allocate(&f->buffer, &allocate_info);
+ gpu_buffer_allocate(&f->buffer, &allocate_info);
}
- vk_buffer_range_upload(&f->buffer, filter, 0, byte_size, 0);
+ gpu_buffer_range_upload(&f->buffer, filter, 0, byte_size, 0);
temp_end(scratch);
}
@@ -309,7 +309,7 @@ beamformer_update_hadamard(BeamformerComputePlan *cp, BeamformerComputeArrayPara
u64 offset = beamformer_compute_array_parameter_offsets[output_field];
u64 size = beamformer_compute_array_parameter_sizes[output_field] / BeamformerMaxHadamardElements;
size *= order * order;
- vk_buffer_range_upload(&cp->array_parameters, hadamard, offset, size, 0);
+ gpu_buffer_range_upload(&cp->array_parameters, hadamard, offset, size, 0);
}
}
@@ -321,8 +321,15 @@ beamformer_frame_byte_size(iv3 points, BeamformerDataKind kind)
return result;
}
+function u64
+beamformer_incoherent_frame_byte_size(iv3 points, BeamformerDataKind kind)
+{
+ u64 result = beamformer_frame_byte_size(points, kind) / beamformer_data_kind_element_count[kind];
+ return result;
+}
+
function BeamformerFrame *
-beamformer_frame_next(BeamformerComputeContext *cc, iv3 output_points, b32 complex, u64 reserved_size)
+beamformer_frame_next(BeamformerComputeContext *cc, iv3 output_points, b32 complex)
{
BeamformerFrameBacklog *bl = &cc->backlog;
@@ -330,9 +337,9 @@ beamformer_frame_next(BeamformerComputeContext *cc, iv3 output_points, b32 compl
u64 frame_size = beamformer_frame_byte_size(output_points, kind);
// TODO(rnp): handle this somewhat gracefully (even it produces garbled output)
- assert(frame_size + reserved_size <= (u64)bl->buffer->size);
+ assert(frame_size <= (u64)bl->buffer->size);
- if (bl->next_offset > (u64)bl->buffer->size - frame_size - reserved_size)
+ if (bl->next_offset > (u64)bl->buffer->size - frame_size)
bl->next_offset = 0;
u64 id = bl->counter++;
@@ -491,7 +498,10 @@ plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, A
};
//////////////////////////////////////
- // NOTE(rnp): First Pass: build initial graph and insert hard layout constraints
+ // NOTE(rnp): First Pass: build initial graph and insert hard layout constraints.
+ // We can also calculate any temporary space we need so we can patch pointers into
+ // bake parameters in the final pass.
+ i64 temporary_buffer_space = 0;
BeamformerComputeGraph graph = {0};
BeamformerComputeGraphNode *root_node = push_compute_graph_node(&graph, BeamformerShaderKind_Count, scratch);
root_node->input_data_kind = input_data_kind;
@@ -555,12 +565,14 @@ plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, A
node->output_stride.x = 1;
node->output_stride.y = cp->output_points.x;
node->output_stride.z = cp->output_points.x * cp->output_points.y;
- node->output_data_kind = cp->iq_pipeline ? BeamformerDataKind_Float32Complex
- : BeamformerDataKind_Float32;
+ node->output_data_kind = das_data_kind;
// NOTE(rnp): insert implicit CoherencyWeighting node
- if (pb->parameters.coherency_weighting)
+ if (pb->parameters.coherency_weighting) {
+ temporary_buffer_space = gpu_round_up_to_sync_size(temporary_buffer_space, 64);
+ temporary_buffer_space += beamformer_incoherent_frame_byte_size(cp->output_points, node->output_data_kind);
node = push_compute_graph_node(&graph, BeamformerShaderKind_CoherencyWeighting, scratch);
+ }
}break;
default:{}break;
@@ -629,6 +641,24 @@ plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, A
cp->first_image_shader_index = 0;
cp->pipeline.shader_count = 0;
+ // NOTE(rnp): realloc temporary buffer if needed. we do want this to shrink
+ // when it can but if we get here and the size didn't change save some time
+ if (temporary_buffer_space != cp->gpu_temp_arena.size) {
+ gpu_buffer_allocate(&cp->gpu_temp_arena, &(GPUBufferAllocateInfo){
+ .size = temporary_buffer_space,
+ .flags = VulkanUsageFlag_TransferDestination,
+ .label = push_str8_f(scratch, "GPU Temp Arena [%p]", cp),
+ });
+ }
+
+ BumpArena gpu_arena = bump_arena_from_buffer((void *)cp->gpu_temp_arena.gpu_pointer, cp->gpu_temp_arena.size);
+ u64 incoherent_buffer = 0;
+ if (pb->parameters.coherency_weighting) {
+ incoherent_buffer = (u64)gpu_arena_alloc(&gpu_arena,
+ .size = beamformer_incoherent_frame_byte_size(cp->output_points, das_data_kind),
+ .align = gpu_round_up_to_sync_size(1, 64));
+ }
+
for (BeamformerComputeGraphNode *node = root_node->next; node; node = node->next) {
assert(node->prev->output_data_kind == node->input_data_kind);
assert(bv3_all(iv3_equal(node->prev->output_stride, node->input_stride)));
@@ -775,6 +805,10 @@ plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, A
db->FocusDepth = pb->parameters.focal_vector.E[1];
db->ReadiGroupCount = pb->parameters.readi_group_count;
db->ArrayParameters = cp->array_parameters.gpu_pointer;
+ db->IncoherentFrame = incoherent_buffer;
+ db->OutputSizeX = cp->output_points.x;
+ db->OutputSizeY = cp->output_points.y;
+ db->OutputSizeZ = cp->output_points.z;
db->TransmitReceiveOrientation = pb->parameters.transmit_receive_orientation;
cp->readi_group = pb->parameters.readi_group;
@@ -803,6 +837,13 @@ plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb, A
case BeamformerShaderKind_CoherencyWeighting:{
sd->layout = layout_for_output(cp->output_points);
sd->dispatch = dispatch_for_output(sd->layout, cp->output_points);
+
+ BeamformerCoherencyWeightingBakeParameters *cw = &sd->bake.CoherencyWeighting;
+ cw->Scale = 1.0f,
+ cw->OutputSizeX = cp->output_points.x,
+ cw->OutputSizeY = cp->output_points.y,
+ cw->OutputSizeZ = cp->output_points.z,
+ cw->IncoherentSum = incoherent_buffer;
}break;
case BeamformerShaderKind_Reshape:{
@@ -1082,7 +1123,7 @@ beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp,
.export = cuda ? &ctx->compute_context.ping_pong_export_handle : 0,
.label = str8("PingPongBuffer"),
};
- vk_buffer_allocate(&ctx->compute_context.ping_pong_buffer, &allocate_info);
+ gpu_buffer_allocate(&ctx->compute_context.ping_pong_buffer, &allocate_info);
BeamformerShaderResourceInfo shader_resource_infos[] = {
{
@@ -1125,7 +1166,7 @@ beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp,
for (u32 i = 0; i < countof(pb->transmit_receive_orientations); i++)
u16s[i] = pb->transmit_receive_orientations[i];
- vk_buffer_range_upload(b, u16s, offset, size, 0);
+ gpu_buffer_range_upload(b, u16s, offset, size, 0);
}
}break;
case BeamformerParameterRegionFlag_FocalVectors:
@@ -1146,7 +1187,7 @@ beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp,
GPUBuffer *b = &cp->array_parameters;
u64 offset = beamformer_compute_array_parameter_offsets[kind];
u64 size = beamformer_compute_array_parameter_sizes[kind];
- vk_buffer_range_upload(b, (u8 *)pb + BeamformerParameterBlockRegionOffsets[region], offset, size, 0);
+ gpu_buffer_range_upload(b, (u8 *)pb + BeamformerParameterBlockRegionOffsets[region], offset, size, 0);
}
}break;
}
@@ -1220,18 +1261,12 @@ do_compute_shader(BeamformerCtx *ctx, GPUCommandList cmd, BeamformerComputePlan
case BeamformerShaderKind_DAS:{
GPUBuffer *b = cc->backlog.buffer;
- u64 frame_size = beamformer_frame_byte_size(frame->points, frame->data_kind);
- u64 iframe_size = frame_size / beamformer_data_kind_element_count[frame->data_kind];
u64 element_size = beamformer_data_kind_byte_size[cp->shader_descriptors[shader_slot].input_data_kind];
BeamformerDASPushConstants pc = {
.xdc_element_pitch = cp->xdc_element_pitch,
.rf_element_offset = das_output_index * pp_size / element_size,
.output_frame = b->gpu_pointer + frame->buffer_offset,
- .incoherent_frame = b->gpu_pointer + b->size - iframe_size,
- .output_size_x = cp->output_points.x,
- .output_size_y = cp->output_points.y,
- .output_size_z = cp->output_points.z,
.channel_offset = channel_offset,
.readi_group = cp->readi_group,
};
@@ -1244,20 +1279,9 @@ do_compute_shader(BeamformerCtx *ctx, GPUCommandList cmd, BeamformerComputePlan
}break;
case BeamformerShaderKind_CoherencyWeighting:{
- GPUBuffer *b = cc->backlog.buffer;
-
- u64 frame_size = beamformer_frame_byte_size(frame->points, frame->data_kind);
- u64 iframe_size = frame_size / beamformer_data_kind_element_count[frame->data_kind];
-
BeamformerCoherencyWeightingPushConstants pc = {
- .left_side_buffer = b->gpu_pointer + frame->buffer_offset,
- .right_side_buffer = b->gpu_pointer + b->size - iframe_size,
- .scale = 1.0f,
- .output_size_x = cp->output_points.x,
- .output_size_y = cp->output_points.y,
- .output_size_z = cp->output_points.z,
+ .coherent_sum = cc->backlog.buffer->gpu_pointer + frame->buffer_offset,
};
-
gpu_command_pipeline_barrier(cmd);
gpu_command_push_constants(cmd, 0, sizeof(pc), &pc);
gpu_command_dispatch_compute(cmd, dispatch);
@@ -1374,7 +1398,7 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
// just fill up as much as possible.
if (exported_size + frame_size <= ec->size) {
gpu_host_wait_timeline(GPUTimeline_Compute, f->timeline_valid_value, -1ULL);
- vk_buffer_range_download(sm_output + exported_size, bl->buffer, f->buffer_offset, frame_size, 1);
+ gpu_buffer_range_download(sm_output + exported_size, bl->buffer, f->buffer_offset, frame_size, 1);
exported_size += frame_size;
}
}
@@ -1438,28 +1462,16 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
start_renderdoc_capture();
i32 das_index = -1;
- b32 has_sum = 0;
+ i32 coherency_weighting = -1;
for (u32 i = 0; i < cp->pipeline.shader_count; i++) {
- has_sum |= cp->pipeline.shaders[i] == BeamformerShaderKind_Sum;
+ if (cp->pipeline.shaders[i] == BeamformerShaderKind_CoherencyWeighting)
+ coherency_weighting = (i32)i;
+
if (cp->pipeline.shaders[i] == BeamformerShaderKind_DAS)
das_index = (i32)i;
}
- b32 das_coherent = das_index >= 0 &&
- (cp->shader_descriptors[das_index].compile_flags &
- BeamformerDASCompileFlags_CoherencyWeighting) != 0;
- u64 reserved_frame_size = 0;
-
- if (has_sum)
- reserved_frame_size += beamformer_frame_byte_size(cp->output_points, cp->iq_pipeline ?
- BeamformerDataKind_Float32Complex :
- BeamformerDataKind_Float32);
-
- // TODO(rnp): incoherent sum for different data kinds
- if (das_coherent)
- reserved_frame_size += beamformer_frame_byte_size(cp->output_points, BeamformerDataKind_Float32);
-
- BeamformerFrame *frame = beamformer_frame_next(cs, cp->output_points, cp->iq_pipeline, reserved_frame_size);
+ BeamformerFrame *frame = beamformer_frame_next(cs, cp->output_points, cp->iq_pipeline);
frame->acquisition_kind = cp->acquisition_kind;
frame->contrast_mode = cp->contrast_mode;
frame->compound_count = cp->acquisition_count;
@@ -1473,12 +1485,14 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
if (das_index >= 0) {
u64 frame_size = beamformer_frame_byte_size(frame->points, frame->data_kind);
GPUBuffer *backlog = cs->backlog.buffer;
-
gpu_command_clear_buffer(cmd, backlog, frame->buffer_offset, frame_size, 0);
- if (das_coherent) {
- u64 coherent_size = frame_size / beamformer_data_kind_element_count[frame->data_kind];
- gpu_command_clear_buffer(cmd, backlog, backlog->size - coherent_size, coherent_size, 0);
- }
+ }
+
+ if (coherency_weighting >= 0) {
+ BeamformerCoherencyWeightingBakeParameters *cw = &cp->shader_descriptors[coherency_weighting].bake.CoherencyWeighting;
+ GPUBuffer *gpu_arena = &cp->gpu_temp_arena;
+ u64 coherent_size = beamformer_incoherent_frame_byte_size(frame->points, frame->data_kind);
+ gpu_command_clear_buffer(cmd, gpu_arena, cw->IncoherentSum - gpu_arena->gpu_pointer, coherent_size, 0);
}
BeamformerRFBuffer *rf = &cs->rf_buffer;
@@ -1553,7 +1567,8 @@ complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena)
cs->processing_progress = 1;
- if (has_sum) {
+ //if (has_sum) {
+ if (0) {
#if 0
u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames));
ctx->averaged_frames[aframe_index].view_plane_tag = frame->view_plane_tag;
@@ -1663,14 +1678,14 @@ DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
BeamformerRFBuffer *rf = ctx->rf_buffer;
- rf->active_rf_size = vk_round_up_to_sync_size(rf_block_rf_size & 0xFFFFFFFFULL, 64);
+ rf->active_rf_size = gpu_round_up_to_sync_size(rf_block_rf_size & 0xFFFFFFFFULL, 64);
if unlikely(rf->buffer.size < countof(rf->upload_complete_values) * rf->active_rf_size) {
GPUBufferAllocateInfo allocate_info = {
.size = countof(rf->upload_complete_values) * rf->active_rf_size,
.flags = VulkanUsageFlag_HostReadWrite,
.label = str8("RawRFBuffer"),
};
- vk_buffer_allocate(&rf->buffer, &allocate_info);
+ gpu_buffer_allocate(&rf->buffer, &allocate_info);
}
u64 slot = rf->insertion_index % countof(rf->upload_complete_values);
@@ -1679,8 +1694,8 @@ DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload)
spin_wait(atomic_load_u64(&rf->compute_index) < rf->insertion_index);
gpu_host_wait_timeline(GPUTimeline_Compute, rf->compute_complete_values[slot], -1ULL);
- vk_buffer_range_upload(&rf->buffer, beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size),
- slot * rf->active_rf_size, rf->active_rf_size, 1);
+ gpu_buffer_range_upload(&rf->buffer, beamformer_shared_memory_data_pointer(sm, ctx->shared_memory_size),
+ slot * rf->active_rf_size, rf->active_rf_size, 1);
store_fence();
beamformer_shared_memory_release_lock(ctx->shared_memory, (i32)scratch_lock);
diff --git a/beamformer_internal.h b/beamformer_internal.h
@@ -18,6 +18,7 @@
#define os_path_separator() (str8){.data = &os_system_info()->path_separator_byte, .length = 1}
+typedef struct { u64 value; } GPUHandle;
typedef struct { u64 value; } GPUCommandList;
typedef struct { u64 value; } GPUSplitBarrier;
@@ -65,12 +66,12 @@ typedef struct {
} VulkanPipelineCreateInfo;
typedef struct {
- VulkanHandle handle;
- u64 gpu_pointer;
- i64 size;
+ GPUHandle handle;
+ u64 gpu_pointer;
+ i64 size;
// NOTE: only used for render models
- u64 index_count;
+ u64 index_count;
} GPUBuffer;
typedef struct {
@@ -131,7 +132,7 @@ typedef struct {
typedef struct {
BeamformerShaderResourceKind kind;
- VulkanHandle handle;
+ GPUHandle handle;
u32 slot;
} BeamformerShaderResourceInfo;
@@ -144,11 +145,11 @@ DEBUG_IMPORT void vk_load(OSLibrary vulkan, Stream *error);
DEBUG_IMPORT GPUInfo *gpu_info(void);
-DEBUG_IMPORT void vk_buffer_allocate(GPUBuffer *, GPUBufferAllocateInfo *info);
-DEBUG_IMPORT void vk_buffer_release(GPUBuffer *);
-DEBUG_IMPORT void vk_buffer_range_upload(GPUBuffer *, void *data, u64 offset, u64 size, b32 non_temporal);
-DEBUG_IMPORT void vk_buffer_range_download(void *output, GPUBuffer *, u64 source_offset, u64 size, b32 non_temporal);
-DEBUG_IMPORT u64 vk_round_up_to_sync_size(u64, u64 min);
+DEBUG_IMPORT void gpu_buffer_allocate(GPUBuffer *, GPUBufferAllocateInfo *info);
+DEBUG_IMPORT void gpu_buffer_release(GPUBuffer *);
+DEBUG_IMPORT void gpu_buffer_range_upload(GPUBuffer *, void *data, u64 offset, u64 size, b32 non_temporal);
+DEBUG_IMPORT void gpu_buffer_range_download(void *output, GPUBuffer *, u64 source_offset, u64 size, b32 non_temporal);
+DEBUG_IMPORT u64 gpu_round_up_to_sync_size(u64, u64 min);
// NOTE: images are 2D only, any other use case should just use a buffer and index in the shader
DEBUG_IMPORT void vk_image_allocate(GPUImage *, u32 width, u32 height, u32 mips, u32 samples, VulkanImageUsage usage, VulkanUsageFlags flags, OSHandle *export, str8 label);
@@ -317,6 +318,7 @@ struct BeamformerComputePlan {
u32 readi_group;
GPUBuffer array_parameters;
+ GPUBuffer gpu_temp_arena;
BeamformerFilter filters[BeamformerFilterSlots];
diff --git a/generated/beamformer.c b/generated/beamformer.c
@@ -208,6 +208,7 @@ typedef struct {
typedef struct {
u64 ArrayParameters;
+ u64 IncoherentFrame;
u32 AcquisitionKind;
b32 Sparse;
i32 AcquisitionCount;
@@ -225,10 +226,21 @@ typedef struct {
b32 SingleFocus;
f32 FocusDepth;
f32 TransmitAngle;
+ u32 OutputSizeX;
+ u32 OutputSizeY;
+ u32 OutputSizeZ;
u32 ReadiGroupCount;
} BeamformerDASBakeParameters;
typedef struct {
+ u64 IncoherentSum;
+ f32 Scale;
+ u32 OutputSizeX;
+ u32 OutputSizeY;
+ u32 OutputSizeZ;
+} BeamformerCoherencyWeightingBakeParameters;
+
+typedef struct {
u32 SizeX;
u32 SizeY;
u32 SizeZ;
@@ -255,11 +267,7 @@ typedef struct {
m4 voxel_transform;
v2 xdc_element_pitch;
u64 output_frame;
- u64 incoherent_frame;
u32 rf_element_offset;
- u32 output_size_x;
- u32 output_size_y;
- u32 output_size_z;
i32 channel_offset;
u32 readi_group;
} BeamformerDASPushConstants;
@@ -272,12 +280,7 @@ typedef struct {
} BeamformerSumPushConstants;
typedef struct {
- u64 left_side_buffer;
- u64 right_side_buffer;
- f32 scale;
- u32 output_size_x;
- u32 output_size_y;
- u32 output_size_z;
+ u64 coherent_sum;
} BeamformerCoherencyWeightingPushConstants;
typedef struct {
@@ -470,10 +473,11 @@ typedef struct {
} BeamformerComputeArrayParameters;
typedef union {
- BeamformerDecodeBakeParameters Decode;
- BeamformerFilterBakeParameters Filter;
- BeamformerDASBakeParameters DAS;
- BeamformerReshapeBakeParameters Reshape;
+ BeamformerDecodeBakeParameters Decode;
+ BeamformerFilterBakeParameters Filter;
+ BeamformerDASBakeParameters DAS;
+ BeamformerCoherencyWeightingBakeParameters CoherencyWeighting;
+ BeamformerReshapeBakeParameters Reshape;
} BeamformerShaderBakeParameters;
read_only global u32 beamformer_compute_array_parameter_sizes[] = {
@@ -611,10 +615,11 @@ read_only global str8 beamformer_shader_resource_kind_strings[] = {
};
typedef enum {
- BeamformerStructKind_DecodeBakeParameters = 0,
- BeamformerStructKind_FilterBakeParameters = 1,
- BeamformerStructKind_DASBakeParameters = 2,
- BeamformerStructKind_ReshapeBakeParameters = 3,
+ BeamformerStructKind_DecodeBakeParameters = 0,
+ BeamformerStructKind_FilterBakeParameters = 1,
+ BeamformerStructKind_DASBakeParameters = 2,
+ BeamformerStructKind_CoherencyWeightingBakeParameters = 3,
+ BeamformerStructKind_ReshapeBakeParameters = 4,
BeamformerStructKind_Count,
} BeamformerStructKind;
@@ -649,24 +654,35 @@ read_only global MetaStructMember *meta_struct_members_by_id[] = {
},
(MetaStructMember []){
{17, 0, 1, 0},
- {18, 8, 1, 0},
- {14, 12, 1, 0},
- {10, 16, 1, 0},
- {10, 20, 1, 0},
+ {17, 8, 1, 0},
+ {18, 16, 1, 0},
+ {14, 20, 1, 0},
{10, 24, 1, 0},
{10, 28, 1, 0},
- {8, 32, 1, 0},
- {8, 36, 1, 0},
+ {10, 32, 1, 0},
+ {10, 36, 1, 0},
{8, 40, 1, 0},
{8, 44, 1, 0},
- {18, 48, 1, 0},
+ {8, 48, 1, 0},
{8, 52, 1, 0},
- {14, 56, 1, 0},
- {18, 60, 1, 0},
+ {18, 56, 1, 0},
+ {8, 60, 1, 0},
{14, 64, 1, 0},
- {8, 68, 1, 0},
- {8, 72, 1, 0},
- {18, 76, 1, 0},
+ {18, 68, 1, 0},
+ {14, 72, 1, 0},
+ {8, 76, 1, 0},
+ {8, 80, 1, 0},
+ {18, 84, 1, 0},
+ {18, 88, 1, 0},
+ {18, 92, 1, 0},
+ {18, 96, 1, 0},
+ },
+ (MetaStructMember []){
+ {17, 0, 1, 0},
+ {8, 8, 1, 0},
+ {18, 12, 1, 0},
+ {18, 16, 1, 0},
+ {18, 20, 1, 0},
},
(MetaStructMember []){
{18, 0, 1, 0},
@@ -712,6 +728,7 @@ read_only global str8 *meta_struct_member_names_by_id[] = {
},
(str8 []){
str8_comp("ArrayParameters"),
+ str8_comp("IncoherentFrame"),
str8_comp("AcquisitionKind"),
str8_comp("Sparse"),
str8_comp("AcquisitionCount"),
@@ -729,9 +746,19 @@ read_only global str8 *meta_struct_member_names_by_id[] = {
str8_comp("SingleFocus"),
str8_comp("FocusDepth"),
str8_comp("TransmitAngle"),
+ str8_comp("OutputSizeX"),
+ str8_comp("OutputSizeY"),
+ str8_comp("OutputSizeZ"),
str8_comp("ReadiGroupCount"),
},
(str8 []){
+ str8_comp("IncoherentSum"),
+ str8_comp("Scale"),
+ str8_comp("OutputSizeX"),
+ str8_comp("OutputSizeY"),
+ str8_comp("OutputSizeZ"),
+ },
+ (str8 []){
str8_comp("SizeX"),
str8_comp("SizeY"),
str8_comp("SizeZ"),
@@ -745,10 +772,11 @@ read_only global str8 *meta_struct_member_names_by_id[] = {
};
read_only global MetaStructInfo meta_struct_info_by_id[] = {
- {str8_comp("DecodeBakeParameters"), 11, 48, 0},
- {str8_comp("FilterBakeParameters"), 13, 56, 0},
- {str8_comp("DASBakeParameters"), 19, 80, 0},
- {str8_comp("ReshapeBakeParameters"), 9, 36, 0},
+ {str8_comp("DecodeBakeParameters"), 11, 48, 0},
+ {str8_comp("FilterBakeParameters"), 13, 56, 0},
+ {str8_comp("DASBakeParameters"), 23, 100, 0},
+ {str8_comp("CoherencyWeightingBakeParameters"), 5, 24, 0},
+ {str8_comp("ReshapeBakeParameters"), 9, 36, 0},
};
read_only global str8 beamformer_shader_names[] = {
@@ -900,11 +928,7 @@ read_only global str8 beamformer_shader_global_header_strings[] = {
" f32mat4 voxel_transform;\n"
" f32vec2 xdc_element_pitch;\n"
" uint64_t output_frame;\n"
- " uint64_t incoherent_frame;\n"
" uint32_t rf_element_offset;\n"
- " uint32_t output_size_x;\n"
- " uint32_t output_size_y;\n"
- " uint32_t output_size_z;\n"
" int32_t channel_offset;\n"
" uint32_t readi_group;\n"
"};\n"
@@ -919,12 +943,7 @@ read_only global str8 beamformer_shader_global_header_strings[] = {
"\n"),
str8_comp(""
"layout(push_constant, std430) uniform PushConstants {\n"
- " uint64_t left_side_buffer;\n"
- " uint64_t right_side_buffer;\n"
- " float32_t scale;\n"
- " uint32_t output_size_x;\n"
- " uint32_t output_size_y;\n"
- " uint32_t output_size_z;\n"
+ " uint64_t coherent_sum;\n"
"};\n"
"\n"),
str8_comp(""
@@ -1040,8 +1059,8 @@ read_only global i32 beamformer_base_shader_to_bake_struct_id[] = {
2,
-1,
-1,
- -1,
3,
+ 4,
-1,
};
diff --git a/shaders/coherency_weighting.glsl b/shaders/coherency_weighting.glsl
@@ -16,26 +16,26 @@ layout(std430, buffer_reference, buffer_reference_align = 8) restrict buffer Flo
};
#if InputDataKind == DataKind_Float32
- #define COHERENT_SAMPLE(index) Float32(left_side_buffer).values[index]
- #define INCOHERENT_SAMPLE(index) Float32(right_side_buffer).values[index]
+ #define COHERENT_SAMPLE(index) Float32(coherent_sum).values[index]
+ #define INCOHERENT_SAMPLE(index) Float32(IncoherentSum).values[index]
#elif InputDataKind == DataKind_Float32Complex
- #define COHERENT_SAMPLE(index) Float32Complex(left_side_buffer).values[index]
- #define INCOHERENT_SAMPLE(index) Float32(right_side_buffer).values[index]
+ #define COHERENT_SAMPLE(index) Float32Complex(coherent_sum).values[index]
+ #define INCOHERENT_SAMPLE(index) Float32(IncoherentSum).values[index]
#else
#error DataKind unsupported for CoherencyWeighting
#endif
-uint32_t output_index(uint32_t x, uint32_t y, uint32_t z)
+u32 output_index(const u32 x, const u32 y, const u32 z)
{
- uint32_t result = output_size_x * output_size_y * z + output_size_x * y + x;
+ u32 result = OutputSizeX * OutputSizeY * z + OutputSizeX * y + x;
return result;
}
void main()
{
uvec3 out_voxel = gl_GlobalInvocationID;
- if (!all(lessThan(out_voxel, uvec3(output_size_x, output_size_y, output_size_z))))
+ if (!all(lessThan(out_voxel, uvec3(OutputSizeX, OutputSizeY, OutputSizeZ))))
return;
- uint32_t index = output_index(out_voxel.x, out_voxel.y, out_voxel.z);
- COHERENT_SAMPLE(index) *= scale * COHERENT_SAMPLE(index) / INCOHERENT_SAMPLE(index);
+ u32 index = output_index(out_voxel.x, out_voxel.y, out_voxel.z);
+ COHERENT_SAMPLE(index) *= Scale * COHERENT_SAMPLE(index) / INCOHERENT_SAMPLE(index);
}
diff --git a/shaders/das.glsl b/shaders/das.glsl
@@ -127,9 +127,9 @@ float sample_index(const float distance)
return time * SamplingFrequency;
}
-uint32_t output_index(uint32_t x, uint32_t y, uint32_t z)
+u32 output_index(const u32 x, const u32 y, const u32 z)
{
- uint32_t result = output_size_x * output_size_y * z + output_size_x * y + x;
+ u32 result = OutputSizeX * OutputSizeY * z + OutputSizeX * y + x;
return result;
}
@@ -368,10 +368,10 @@ RESULT_TYPE READI_FORCES(const vec3 xdc_world_point)
void main()
{
uvec3 out_voxel = gl_GlobalInvocationID;
- if (!all(lessThan(out_voxel, uvec3(output_size_x, output_size_y, output_size_z))))
+ if (!all(lessThan(out_voxel, uvec3(OutputSizeX, OutputSizeY, OutputSizeZ))))
return;
- vec3 image_points = vec3(output_size_x, output_size_y, output_size_z) - 1.0f;
+ vec3 image_points = vec3(OutputSizeX, OutputSizeY, OutputSizeZ) - 1.0f;
vec3 point = vec3(out_voxel) / max(vec3(1.0f), image_points);
vec3 world_point = (voxel_transform * vec4(point, 1)).xyz;
@@ -400,7 +400,7 @@ void main()
}
#if CoherencyWeighting
- IncoherentOutput(incoherent_frame).x[out_index] += RESULT_INCOHERENT_CAST(sum);
+ IncoherentOutput(IncoherentFrame).x[out_index] += RESULT_INCOHERENT_CAST(sum);
#endif
Output(output_frame).x[out_index] += RESULT_COHERENT_CAST(sum);
diff --git a/ui.c b/ui.c
@@ -829,7 +829,7 @@ function void
beamformer_ui_frame_view_release_subresources(BeamformerFrameView *bv, BeamformerFrameViewKind kind)
{
if (kind == BeamformerFrameViewKind_Copy)
- vk_buffer_release(&bv->copy_buffer);
+ gpu_buffer_release(&bv->copy_buffer);
}
function void
@@ -851,7 +851,7 @@ beamformer_ui_frame_view_copy_frame(BeamformerFrameView *new, BeamformerFrameVie
.flags = VulkanUsageFlag_TransferDestination,
.label = stream_to_str8(&sb),
};
- vk_buffer_allocate(&new->copy_buffer, &allocate_info);
+ gpu_buffer_allocate(&new->copy_buffer, &allocate_info);
GPUBuffer *backlog = beamformer_context->compute_context.backlog.buffer;
GPUCommandList cmd = gpu_command_list_begin(GPUTimeline_Compute);
diff --git a/util.c b/util.c
@@ -120,13 +120,19 @@ round_up_to(i64 value, i64 multiple)
return result;
}
-function u8 *
-arena_commit(Arena *a, i64 size)
+function BumpArena
+bump_arena_from_buffer(void *base, i64 size)
{
- Arena *current = a->current;
- assert(current->committed - current->position >= (u64)size);
- u8 *result = (u8 *)current + current->position;
- current->position += size;
+ BumpArena result = {.start = base};
+ result.end = result.start + size;
+ return result;
+}
+
+function void *
+bump_arena_aligned_start(BumpArena a, u64 alignment)
+{
+ u64 padding = -(u64)a.start & (alignment - 1);
+ u8 *result = a.start + padding;
return result;
}
@@ -141,6 +147,39 @@ typedef struct {
ArenaAllocateFlags flags;
} ArenaAllocateInfo;
+#define bump_arena_alloc(a, ...) bump_arena_alloc_(a, (ArenaAllocateInfo){.align = 8, .count = 1, ##__VA_ARGS__})
+#define gpu_arena_alloc(a, ...) bump_arena_alloc_(a, (ArenaAllocateInfo){.flags = ArenaAllocateFlags_NoZero, .align = 8, .count = 1, ##__VA_ARGS__})
+#define bump_push_array(a, t, n) (t *)bump_arena_alloc(a, .size = sizeof(t), .align = alignof(t), .count = n)
+#define bump_push_array_no_zero(a, t, n) (t *)bump_arena_alloc(a, .size = sizeof(t), .align = alignof(t), .count = n, .flags = ArenaAllocateFlags_NoZero)
+#define bump_push_struct(a, t) bump_push_array(a, t, 1)
+#define bump_push_struct_no_zero(a, t) bump_push_array_no_zero(a, t, 1)
+
+function void *
+bump_arena_alloc_(BumpArena *a, ArenaAllocateInfo info)
+{
+ void *result = 0;
+ if (a->start) {
+ u8 *start = bump_arena_aligned_start(*a, info.align);
+ i64 available = a->end - start;
+ assert((available >= 0 && info.count <= available / info.size));
+ a->start = start + info.count * info.size;
+ result = start;
+ if ((info.flags & ArenaAllocateFlags_NoZero) == 0)
+ result = memory_clear(start, 0, info.count * info.size);
+ }
+ return result;
+}
+
+function u8 *
+arena_commit(Arena *a, i64 size)
+{
+ Arena *current = a->current;
+ assert(current->committed - current->position >= (u64)size);
+ u8 *result = (u8 *)current + current->position;
+ current->position += size;
+ return result;
+}
+
#define arena_alloc(a, ...) arena_alloc_(a, (ArenaAllocateInfo){.align = 8, .count = 1, ##__VA_ARGS__})
#define push_array(a, t, n) (t *)arena_alloc(a, .size = sizeof(t), .align = alignof(t), .count = n)
#define push_array_no_zero(a, t, n) (t *)arena_alloc(a, .size = sizeof(t), .align = alignof(t), .count = n, .flags = ArenaAllocateFlags_NoZero)
@@ -990,9 +1029,24 @@ push_str8(Arena *a, str8 str)
function str8
push_str8_fv(Arena *arena, const char *format, va_list args)
{
- Stream sb = arena_stream(arena);
- stream_appendfv(&sb, format, args);
- str8 result = arena_stream_commit(arena, &sb);
+ va_list args2;
+ va_copy(args2, args);
+ i64 size = vsnprintf(0, 0, format, args2);
+ va_end(args2);
+ str8 result = str8_alloc(arena, size + 1);
+ i64 written = vsnprintf((char *)result.data, result.length, format, args);
+ assert(written == size);
+ result.length -= 1;
+ return result;
+}
+
+function print_format(2, 3) str8
+push_str8_f(Arena *arena, const char *format, ...)
+{
+ va_list args;
+ va_start(args, format);
+ str8 result = push_str8_fv(arena, format, args);
+ va_end(args);
return result;
}
diff --git a/vulkan.c b/vulkan.c
@@ -1972,19 +1972,19 @@ vk_vulkan_buffer_release(VulkanBuffer *vb)
}
DEBUG_IMPORT void
-vk_buffer_release(GPUBuffer *b)
+gpu_buffer_release(GPUBuffer *b)
{
- if ValidVulkanHandle(b->handle)
- vk_vulkan_buffer_release(vk_entity_data(b->handle.value[0], VulkanEntityKind_Buffer));
+ if (b->handle.value)
+ vk_vulkan_buffer_release(vk_entity_data(b->handle.value, VulkanEntityKind_Buffer));
zero_struct(b);
}
DEBUG_IMPORT void
-vk_buffer_allocate(GPUBuffer *b, GPUBufferAllocateInfo *info)
+gpu_buffer_allocate(GPUBuffer *b, GPUBufferAllocateInfo *info)
{
VulkanContext *vk = vulkan_context;
- vk_buffer_release(b);
+ gpu_buffer_release(b);
assert(info->size > 0);
@@ -2010,7 +2010,7 @@ vk_buffer_allocate(GPUBuffer *b, GPUBufferAllocateInfo *info)
}
if (vk_buffer_allocate_common(&e->as.buffer, &vulkan_buffer_allocate_info)) {
- b->handle.value[0] = (u64)e;
+ b->handle.value = (u64)e;
} else {
vk_entity_release(e);
}
@@ -2020,8 +2020,8 @@ DEBUG_IMPORT b32
vk_buffer_needs_sync(GPUBuffer *b)
{
b32 result = 0;
- if ValidVulkanHandle(b->handle) {
- VulkanBuffer *vb = vk_entity_data(b->handle.value[0], VulkanEntityKind_Buffer);
+ if (b->handle.value) {
+ VulkanBuffer *vb = vk_entity_data(b->handle.value, VulkanEntityKind_Buffer);
// TODO(rnp): not correct check. need to check if we used transfer queue
result = vb->memory_kind != VulkanMemoryKind_BAR;
@@ -2031,7 +2031,7 @@ vk_buffer_needs_sync(GPUBuffer *b)
}
DEBUG_IMPORT u64
-vk_round_up_to_sync_size(u64 size, u64 min)
+gpu_round_up_to_sync_size(u64 size, u64 min)
{
i64 round = (i64)Max(min, vulkan_context->memory_info.non_coherent_atom_size);
u64 result = (u64)round_up_to((i64)size, round);
@@ -2060,7 +2060,7 @@ vk_buffer_buffer_copy(VulkanBuffer *destination, VulkanBuffer *source, u64 desti
.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
.memory = source->memory,
.offset = source_offset - (source_offset % nca_size),
- .size = vk_round_up_to_sync_size(size, nca_size),
+ .size = gpu_round_up_to_sync_size(size, nca_size),
}};
vkInvalidateMappedMemoryRanges(vk->device, countof(mrs), mrs);
}
@@ -2096,7 +2096,7 @@ vk_buffer_buffer_copy(VulkanBuffer *destination, VulkanBuffer *source, u64 desti
.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
.memory = destination->memory,
.offset = destination_offset - (destination_offset % nca_size),
- .size = vk_round_up_to_sync_size(size, nca_size),
+ .size = gpu_round_up_to_sync_size(size, nca_size),
}};
vkFlushMappedMemoryRanges(vk->device, countof(mrs), mrs);
}
@@ -2112,9 +2112,9 @@ vk_buffer_buffer_copy(VulkanBuffer *destination, VulkanBuffer *source, u64 desti
}
DEBUG_IMPORT void
-vk_buffer_range_upload(GPUBuffer *b, void *data, u64 offset, u64 size, b32 non_temporal)
+gpu_buffer_range_upload(GPUBuffer *b, void *data, u64 offset, u64 size, b32 non_temporal)
{
- VulkanBuffer *db = vk_entity_data(b->handle.value[0], VulkanEntityKind_Buffer);
+ VulkanBuffer *db = vk_entity_data(b->handle.value, VulkanEntityKind_Buffer);
VulkanBuffer sb = {
.host_pointer = data,
.memory_kind = VulkanMemoryKind_Host,
@@ -2123,9 +2123,9 @@ vk_buffer_range_upload(GPUBuffer *b, void *data, u64 offset, u64 size, b32 non_t
}
DEBUG_IMPORT void
-vk_buffer_range_download(void *destination, GPUBuffer *source, u64 offset, u64 size, b32 non_temporal)
+gpu_buffer_range_download(void *destination, GPUBuffer *source, u64 offset, u64 size, b32 non_temporal)
{
- VulkanBuffer *sb = vk_entity_data(source->handle.value[0], VulkanEntityKind_Buffer);
+ VulkanBuffer *sb = vk_entity_data(source->handle.value, VulkanEntityKind_Buffer);
VulkanBuffer db = {
.host_pointer = destination,
.memory_kind = VulkanMemoryKind_Host,
@@ -2136,8 +2136,8 @@ vk_buffer_range_download(void *destination, GPUBuffer *source, u64 offset, u64 s
DEBUG_IMPORT void
vk_render_model_release(GPUBuffer *model)
{
- if ValidVulkanHandle(model->handle)
- vk_vulkan_buffer_release(vk_entity_data(model->handle.value[0], VulkanEntityKind_RenderModel));
+ if (model->handle.value)
+ vk_vulkan_buffer_release(vk_entity_data(model->handle.value, VulkanEntityKind_RenderModel));
zero_struct(model);
}
@@ -2168,7 +2168,7 @@ vk_render_model_allocate(GPUBuffer *model, void *indices, u64 index_count, u64 m
.queue_family_indices[0] = vulkan_context->queues[VulkanQueueKind_Graphics]->queue_family,
};
if (vk_buffer_allocate_common(&e->as.buffer, &vulkan_buffer_allocate_info)) {
- model->handle.value[0] = (u64)e;
+ model->handle.value = (u64)e;
model->index_count = index_count;
model->gpu_pointer += indices_size;
@@ -2186,7 +2186,7 @@ vk_render_model_allocate(GPUBuffer *model, void *indices, u64 index_count, u64 m
DEBUG_IMPORT void
vk_render_model_range_upload(GPUBuffer *model, void *data, u64 offset, u64 size, b32 non_temporal)
{
- VulkanBuffer *db = vk_entity_data(model->handle.value[0], VulkanEntityKind_RenderModel);
+ VulkanBuffer *db = vk_entity_data(model->handle.value, VulkanEntityKind_RenderModel);
VulkanBuffer sb = {
.host_pointer = data,
.memory_kind = VulkanMemoryKind_Host,
@@ -2442,7 +2442,7 @@ vk_bind_shader_resources(BeamformerShaderResourceInfo *infos, u64 info_count)
for EachIndex(info_count, it) {
switch (infos[it].kind) {
case BeamformerShaderResourceKind_Buffer:{
- VulkanBuffer *vb = vk_entity_data(infos[it].handle.value[0], VulkanEntityKind_Buffer);
+ VulkanBuffer *vb = vk_entity_data(infos[it].handle.value, VulkanEntityKind_Buffer);
vk->descriptor_buffer_infos[infos[it].slot].buffer = vb->buffer;
vk->descriptor_buffer_infos[infos[it].slot].offset = 0;
vk->descriptor_buffer_infos[infos[it].slot].range = vb->memory_size;
@@ -2564,7 +2564,7 @@ gpu_command_clear_buffer(GPUCommandList command, GPUBuffer *buffer, u64 offset,
assert((offset % 4) == 0);
assert((size % 4) == 0);
if (command.value) {
- VulkanBuffer *vb = vk_entity_data(buffer->handle.value[0], VulkanEntityKind_Buffer);
+ VulkanBuffer *vb = vk_entity_data(buffer->handle.value, VulkanEntityKind_Buffer);
VkCommandBuffer cmd = vk_command_buffer(command);
vkCmdFillBuffer(cmd, vb->buffer, offset, size, clear_word);
}
@@ -2837,9 +2837,9 @@ gpu_command_begin_rendering(GPUCommandList command, GPUImage *colour, GPUImage *
DEBUG_IMPORT void
gpu_command_draw(GPUCommandList command, GPUBuffer *model)
{
- if (command.value && ValidVulkanHandle(model->handle)) {
+ if (command.value && model->handle.value) {
VkCommandBuffer cmd = vk_command_buffer(command);
- VulkanBuffer *vb = vk_entity_data(model->handle.value[0], VulkanEntityKind_RenderModel);
+ VulkanBuffer *vb = vk_entity_data(model->handle.value, VulkanEntityKind_RenderModel);
vkCmdBindIndexBuffer2(cmd, vb->buffer, 0, vk_index_size(vb->index_type) * model->index_count, vb->index_type);
vkCmdDrawIndexed(cmd, model->index_count, 1, 0, 0, 0);
}
@@ -2875,10 +2875,10 @@ DEBUG_IMPORT void
gpu_command_copy_buffer(GPUCommandList command, GPUBuffer *restrict destination,
GPUBuffer *restrict source, u64 source_offset, i64 size)
{
- if (command.value && ValidVulkanHandle(destination->handle) && ValidVulkanHandle(source->handle)) {
+ if (command.value && destination->handle.value && source->handle.value) {
VkCommandBuffer cmd = vk_command_buffer(command);
- VulkanBuffer *db = vk_entity_data(destination->handle.value[0], VulkanEntityKind_Buffer);
- VulkanBuffer *sb = vk_entity_data(source->handle.value[0], VulkanEntityKind_Buffer);
+ VulkanBuffer *db = vk_entity_data(destination->handle.value, VulkanEntityKind_Buffer);
+ VulkanBuffer *sb = vk_entity_data(source->handle.value, VulkanEntityKind_Buffer);
VkBufferCopy2 buffer_copy = {
.sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2,