beamformer.c (12724B)
1 /* See LICENSE for license details. */ 2 3 #include "beamformer_internal.h" 4 5 /* NOTE(rnp): magic variables to force discrete GPU usage on laptops with multiple devices */ 6 EXPORT i32 NvOptimusEnablement = 1; 7 EXPORT i32 AmdPowerXpressRequestHighPerformance = 1; 8 9 #if !BEAMFORMER_DEBUG 10 #include "beamformer_core.c" 11 #else 12 13 typedef void beamformer_frame_step_fn(void *, BeamformerInput *); 14 15 #define BEAMFORMER_DEBUG_ENTRY_POINTS \ 16 X(beamformer_debug_ui_deinit) \ 17 X(beamformer_complete_compute) \ 18 X(beamformer_frame_step) \ 19 X(beamformer_rf_upload) \ 20 21 #define X(name) global name ##_fn *name; 22 BEAMFORMER_DEBUG_ENTRY_POINTS 23 #undef X 24 25 BEAMFORMER_EXPORT void 26 beamformer_debug_hot_release(void *memory, BeamformerInput *input) 27 { 28 BeamformerCtx *ctx = memory; 29 // TODO(rnp): this will deadlock if live imaging is active 30 /* NOTE(rnp): spin until compute thread finishes its work (we will probably 31 * never reload while compute is in progress but just incase). */ 32 spin_wait(atomic_load_u32(&ctx->upload_worker.awake)); 33 spin_wait(atomic_load_u32(&ctx->compute_worker.awake)); 34 } 35 36 BEAMFORMER_EXPORT void 37 beamformer_debug_hot_reload(OSLibrary library) 38 { 39 #define X(name) name = os_lookup_symbol(library, #name); 40 BEAMFORMER_DEBUG_ENTRY_POINTS 41 #undef X 42 43 str8 info = beamformer_info("reloaded main executable"); 44 os_console_log(info.data, info.length); 45 } 46 47 #endif /* BEAMFORMER_DEBUG */ 48 49 function no_return void 50 fatal(str8 message) 51 { 52 os_fatal(message.data, message.length); 53 unreachable(); 54 } 55 56 #include "vulkan.c" 57 58 // TODO(rnp): this doesn't belong here, but will be removed 59 // once vulkan migration is complete 60 void * glfwGetProcAddress(char *); 61 62 function void 63 gl_debug_logger(u32 src, u32 type, u32 id, u32 lvl, i32 len, const char *msg, const void *userctx) 64 { 65 Stream *e = (Stream *)userctx; 66 stream_append_str8s(e, str8("[OpenGL] "), (str8){.length = len, .data = (u8 *)msg}, str8("\n")); 67 os_console_log(e->data, e->widx); 68 stream_reset(e, 0); 69 } 70 71 function void 72 load_gl(Stream *err) 73 { 74 #define X(name, ret, params) name = (name##_fn *)glfwGetProcAddress(#name); 75 OGLProcedureList 76 OGLRequiredExtensionProcedureList 77 #undef X 78 79 stream_reset(err, 0); 80 #define X(name, ret, params) if (!name) stream_append_str8(err, str8("missing required GL function: " #name "\n")); 81 OGLProcedureList 82 OGLRequiredExtensionProcedureListBase 83 #if OS_WINDOWS 84 OGLRequiredExtensionProcedureListW32 85 #else 86 OGLRequiredExtensionProcedureListLinux 87 #endif 88 #undef X 89 90 if (err->widx) fatal(stream_to_str8(err)); 91 } 92 93 function void 94 beamformer_load_cuda_library(BeamformerCtx *ctx, OSLibrary cuda, Arena *scratch) 95 { 96 /* TODO(rnp): (25.10.30) registering the rf buffer with CUDA is currently 97 * causing a major performance regression. for now we are disabling its use 98 * altogether. it will be reenabled once the issue can be fixed */ 99 b32 result = 0 && gpu_info()->vendor == GPUVendor_NVIDIA && ValidHandle(cuda); 100 if (result) { 101 Stream err = arena_stream(scratch); 102 103 stream_append_str8(&err, beamformer_info("loading CUDA library functions")); 104 #define X(name, symname) cuda_## name = os_lookup_symbol(cuda, symname); 105 CUDALibraryProcedureList 106 #undef X 107 108 os_console_log(err.data, err.widx); 109 } 110 111 #define X(name, symname) if (!cuda_## name) cuda_## name = cuda_ ## name ## _stub; 112 CUDALibraryProcedureList 113 #undef X 114 } 115 116 function void 117 worker_thread_sleep(GLWorkerThreadContext *ctx, BeamformerSharedMemory *sm) 118 { 119 for (;;) { 120 i32 expected = 0; 121 if (atomic_cas_u32(&ctx->sync_variable, &expected, 1) || 122 atomic_load_u32(&sm->live_imaging_parameters.active)) 123 { 124 break; 125 } 126 127 /* TODO(rnp): clean this crap up; we shouldn't need two values to communicate this */ 128 atomic_store_u32(&ctx->awake, 0); 129 os_wait_on_address(&ctx->sync_variable, 1, (u32)-1); 130 atomic_store_u32(&ctx->awake, 1); 131 } 132 } 133 134 function OS_THREAD_ENTRY_POINT_FN(compute_worker_thread_entry_point) 135 { 136 GLWorkerThreadContext *ctx = user_context; 137 138 BeamformerCtx *beamformer = (BeamformerCtx *)ctx->user_context; 139 140 for (;;) { 141 worker_thread_sleep(ctx, beamformer->shared_memory); 142 beamformer_complete_compute(beamformer, ctx->arena); 143 } 144 145 unreachable(); 146 147 return 0; 148 } 149 150 function OS_THREAD_ENTRY_POINT_FN(beamformer_upload_entry_point) 151 { 152 GLWorkerThreadContext *ctx = user_context; 153 BeamformerUploadThreadContext *up = (typeof(up))ctx->user_context; 154 155 for (;;) { 156 worker_thread_sleep(ctx, up->shared_memory); 157 beamformer_rf_upload(up); 158 } 159 160 unreachable(); 161 162 return 0; 163 } 164 165 BEAMFORMER_EXPORT void * 166 beamformer_init(BeamformerInput *input) 167 { 168 Arena *memory = arena_create(.name = "Beamformer Memory"); 169 Stream error = stream_alloc(memory, MB(1)); 170 BeamformerCtx *ctx = push_struct(memory, BeamformerCtx); 171 172 for EachElement(ctx->frame_arenas, it) 173 ctx->frame_arenas[it] = arena_create(); 174 175 str8 window_title = str8("VK Beamformer"); 176 ctx->main_window = os_window_create(window_title.data, window_title.length, 1280, 840); 177 ctx->window_size = (iv2){{1280, 840}}; 178 179 ctx->arena = memory; 180 ctx->error_stream = error; 181 ctx->ui_arena = arena_create(); 182 ctx->compute_worker.arena = arena_create(); 183 ctx->upload_worker.arena = arena_create(); 184 185 #if BEAMFORMER_RENDERDOC_HOOKS 186 start_frame_capture = input->renderdoc_start_frame_capture; 187 end_frame_capture = input->renderdoc_end_frame_capture; 188 #endif 189 190 vk_load(input->vulkan_library_handle, &ctx->error_stream); 191 192 BeamformerComputeContext *cs = &ctx->compute_context; 193 194 // NOTE(rnp): allocate beamformed image ring buffer 195 { 196 u64 gpu_heap_size = gpu_info()->gpu_heap_size; 197 u64 trial_sizes[] = { 198 GB(4), 199 GB(2), 200 GB(1) + MB(512), 201 GB(1), 202 }; 203 204 u32 base_index = 0; 205 for EachElement(trial_sizes, it) { 206 if (gpu_heap_size >= 2 * trial_sizes[it]) 207 break; 208 base_index++; 209 } 210 211 for (u32 i = base_index; i < countof(trial_sizes); i++) { 212 // TODO(rnp): it may be better to download data from this using the transfer queue 213 GPUTimeline timelines[] = {GPUTimeline_Compute, GPUTimeline_Graphics}; 214 GPUBufferAllocateInfo allocate_info = { 215 .size = trial_sizes[i], 216 .flags = VulkanUsageFlag_TransferDestination|VulkanUsageFlag_TransferSource|VulkanUsageFlag_HostReadWrite, 217 .timeline_count = countof(timelines), 218 .timelines_used = timelines, 219 .label = str8("BeamformedData"), 220 }; 221 gpu_buffer_allocate(cs->backlog.buffer, allocate_info); 222 if (cs->backlog.buffer->size > 0) 223 break; 224 } 225 if (cs->backlog.buffer->size == 0) { 226 // NOTE(rnp): if this becomes an issue we may be able to get by in some other way 227 fatal(str8("Failed to allocate space for beamformed data\n")); 228 } 229 } 230 231 Arena *scratch = arena_create(); 232 beamformer_load_cuda_library(ctx, input->cuda_library_handle, scratch); 233 234 load_gl(&ctx->error_stream); 235 236 ctx->shared_memory = input->shared_memory; 237 ctx->shared_memory_size = input->shared_memory_size; 238 if (ctx->shared_memory_size < (i64)sizeof(*ctx->shared_memory)) 239 fatal(str8("Get more ram lol\n")); 240 zero_struct(ctx->shared_memory); 241 242 ctx->shared_memory->version = BEAMFORMER_SHARED_MEMORY_VERSION; 243 ctx->shared_memory->reserved_parameter_blocks = 1; 244 245 ctx->shared_memory->beamformed_frame_buffer_size = cs->backlog.buffer->size; 246 247 // TODO(rnp): dynamic rf data buffer slot usage 248 // NOTE(rnp): will be same as the max size we were able to get for the frame buffer 249 ctx->shared_memory->capabilities.max_rf_data_size = cs->backlog.buffer->size 250 / BeamformerMaxRawDataFramesInFlight; 251 252 ctx->shared_memory->capabilities.cuda = cuda_supported(); 253 // TODO(rnp): re-enable hilbert support, with and without cuda 254 ctx->shared_memory->capabilities.hilbert = 0; 255 256 /* TODO(rnp): I'm not sure if its a good idea to pre-reserve a bunch of semaphores 257 * on w32 but thats what we are doing for now */ 258 #if OS_WINDOWS 259 { 260 Stream sb = arena_stream(memory); 261 stream_append(&sb, input->shared_memory_name, input->shared_memory_name_length); 262 stream_append_str8(&sb, str8("_lock_")); 263 i32 start_index = sb.widx; 264 for EachElement(os_w32_shared_memory_semaphores, it) { 265 stream_reset(&sb, start_index); 266 stream_append_u64(&sb, it); 267 stream_append_byte(&sb, 0); 268 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 269 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 270 fatal(beamformer_info("init: failed to create w32 shared memory semaphore\n")); 271 272 /* NOTE(rnp): hacky garbage because CreateSemaphore will just open an existing 273 * semaphore without any indication. Sometimes the other side of the shared memory 274 * will provide incorrect parameters or will otherwise fail and its faster to 275 * restart this program than to get that application to release the semaphores */ 276 /* TODO(rnp): figure out something more robust */ 277 os_w32_semaphore_release(os_w32_shared_memory_semaphores[it], 1); 278 } 279 } 280 #endif 281 282 GLWorkerThreadContext *worker = &ctx->compute_worker; 283 /* TODO(rnp): we should lock this down after we have something working */ 284 worker->user_context = (iptr)ctx; 285 worker->handle = os_create_thread("[compute]", worker, compute_worker_thread_entry_point); 286 287 GLWorkerThreadContext *upload = &ctx->upload_worker; 288 BeamformerUploadThreadContext *upctx = push_struct(memory, typeof(*upctx)); 289 upload->user_context = (iptr)upctx; 290 upctx->rf_buffer = &cs->rf_buffer; 291 upctx->shared_memory = ctx->shared_memory; 292 upctx->shared_memory_size = ctx->shared_memory_size; 293 upctx->compute_timing_table = ctx->compute_timing_table; 294 upctx->compute_worker_sync = &ctx->compute_worker.sync_variable; 295 upload->handle = os_create_thread("[upload]", upload, beamformer_upload_entry_point); 296 297 /* NOTE: set up OpenGL debug logging */ 298 Stream *gl_error_stream = push_struct(memory, Stream); 299 *gl_error_stream = stream_alloc(memory, 1024); 300 glDebugMessageCallback(gl_debug_logger, gl_error_stream); 301 #ifdef BEAMFORMER_DEBUG 302 glEnable(GL_DEBUG_OUTPUT); 303 #endif 304 305 if (!BakeShaders) 306 { 307 for EachElement(beamformer_reloadable_compute_shader_info_indices, it) { 308 i32 index = beamformer_reloadable_compute_shader_info_indices[it]; 309 310 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 311 beamformer_reloadable_shader_files[index][0]); 312 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 313 frc->kind = BeamformerFileReloadKind_ComputeShader; 314 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 315 os_add_file_watch((char *)file.data, file.length, frc); 316 } 317 318 for EachElement(beamformer_reloadable_compute_helpers_shader_info_indices, it) { 319 i32 index = beamformer_reloadable_compute_helpers_shader_info_indices[it]; 320 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 321 beamformer_reloadable_shader_files[index][0]); 322 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 323 frc->kind = BeamformerFileReloadKind_ComputeShader; 324 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 325 os_add_file_watch((char *)file.data, file.length, frc); 326 } 327 } 328 329 arena_destroy(scratch); 330 331 ctx->state = BeamformerState_Running; 332 333 return ctx; 334 } 335 336 BEAMFORMER_EXPORT void 337 beamformer_terminate(void *memory, BeamformerInput *input) 338 { 339 /* NOTE(rnp): work around pebkac when the beamformer is closed while we are doing live 340 * imaging. if the verasonics is blocked in an external function (calling the library 341 * to start compute) it is impossible for us to get it to properly shut down which 342 * will sometimes result in us needing to power cycle the system. set the shared memory 343 * into an error state and release dispatch lock so that future calls will error instead 344 * of blocking. 345 */ 346 BeamformerCtx *ctx = memory; 347 BeamformerSharedMemory *sm = input->shared_memory; 348 if (ctx->state != BeamformerState_Terminated) { 349 if (sm) { 350 BeamformerSharedMemoryLockKind lock = BeamformerSharedMemoryLockKind_DispatchCompute; 351 atomic_store_u32(&sm->invalid, 1); 352 atomic_store_u32(&sm->external_work_queue.ridx, sm->external_work_queue.widx); 353 DEBUG_DECL(if (sm->locks[lock])) { 354 beamformer_shared_memory_release_lock(sm, (i32)lock); 355 } 356 357 atomic_or_u32(&sm->live_imaging_dirty_flags, BeamformerLiveImagingDirtyFlags_StopImaging); 358 } 359 360 beamformer_debug_ui_deinit(ctx); 361 362 ctx->state = BeamformerState_Terminated; 363 } 364 } 365 366 BEAMFORMER_EXPORT u32 367 beamformer_should_close(void *memory, BeamformerInput *input) 368 { 369 BeamformerCtx *ctx = memory; 370 if (ctx->state == BeamformerState_ShouldClose) 371 beamformer_terminate(memory, input); 372 return ctx->state == BeamformerState_Terminated; 373 }