beamformer.c (12794B)
1 /* See LICENSE for license details. */ 2 3 #include "beamformer_internal.h" 4 5 /* NOTE(rnp): magic variables to force discrete GPU usage on laptops with multiple devices */ 6 EXPORT i32 NvOptimusEnablement = 1; 7 EXPORT i32 AmdPowerXpressRequestHighPerformance = 1; 8 9 #if !BEAMFORMER_DEBUG 10 #include "beamformer_core.c" 11 #else 12 13 typedef void beamformer_frame_step_fn(void *, BeamformerInput *); 14 15 #define BEAMFORMER_DEBUG_ENTRY_POINTS \ 16 X(beamformer_debug_ui_deinit) \ 17 X(beamformer_complete_compute) \ 18 X(beamformer_frame_step) \ 19 X(beamformer_rf_upload) \ 20 21 #define X(name) global name ##_fn *name; 22 BEAMFORMER_DEBUG_ENTRY_POINTS 23 #undef X 24 25 BEAMFORMER_EXPORT void 26 beamformer_debug_hot_release(void *memory, BeamformerInput *input) 27 { 28 BeamformerCtx *ctx = memory; 29 // TODO(rnp): this will deadlock if live imaging is active 30 /* NOTE(rnp): spin until compute thread finishes its work (we will probably 31 * never reload while compute is in progress but just incase). */ 32 spin_wait(atomic_load_u32(&ctx->upload_worker.awake)); 33 spin_wait(atomic_load_u32(&ctx->compute_worker.awake)); 34 } 35 36 BEAMFORMER_EXPORT void 37 beamformer_debug_hot_reload(OSLibrary library) 38 { 39 #define X(name) name = os_lookup_symbol(library, #name); 40 BEAMFORMER_DEBUG_ENTRY_POINTS 41 #undef X 42 43 str8 info = beamformer_info("reloaded main executable"); 44 os_console_log(info.data, info.length); 45 } 46 47 #endif /* BEAMFORMER_DEBUG */ 48 49 function no_return void 50 fatal(str8 message) 51 { 52 os_fatal(message.data, message.length); 53 unreachable(); 54 } 55 56 #include "vulkan.c" 57 58 // TODO(rnp): this doesn't belong here, but will be removed 59 // once vulkan migration is complete 60 void * glfwGetProcAddress(char *); 61 62 function void 63 gl_debug_logger(u32 src, u32 type, u32 id, u32 lvl, i32 len, const char *msg, const void *userctx) 64 { 65 Stream *e = (Stream *)userctx; 66 stream_append_str8s(e, str8("[OpenGL] "), (str8){.length = len, .data = (u8 *)msg}, str8("\n")); 67 os_console_log(e->data, e->widx); 68 stream_reset(e, 0); 69 } 70 71 function void 72 load_gl(Stream *err) 73 { 74 #define X(name, ret, params) name = (name##_fn *)glfwGetProcAddress(#name); 75 OGLProcedureList 76 OGLRequiredExtensionProcedureList 77 #undef X 78 79 stream_reset(err, 0); 80 #define X(name, ret, params) if (!name) stream_append_str8(err, str8("missing required GL function: " #name "\n")); 81 OGLProcedureList 82 OGLRequiredExtensionProcedureListBase 83 #if OS_WINDOWS 84 OGLRequiredExtensionProcedureListW32 85 #else 86 OGLRequiredExtensionProcedureListLinux 87 #endif 88 #undef X 89 90 if (err->widx) fatal(stream_to_str8(err)); 91 } 92 93 function void 94 beamformer_load_cuda_library(BeamformerCtx *ctx, OSLibrary cuda, Arena *scratch) 95 { 96 /* TODO(rnp): (25.10.30) registering the rf buffer with CUDA is currently 97 * causing a major performance regression. for now we are disabling its use 98 * altogether. it will be reenabled once the issue can be fixed */ 99 b32 result = 0 && gpu_info()->vendor == GPUVendor_NVIDIA && ValidHandle(cuda); 100 if (result) { 101 Stream err = arena_stream(scratch); 102 103 stream_append_str8(&err, beamformer_info("loading CUDA library functions")); 104 #define X(name, symname) cuda_## name = os_lookup_symbol(cuda, symname); 105 CUDALibraryProcedureList 106 #undef X 107 108 os_console_log(err.data, err.widx); 109 } 110 111 #define X(name, symname) if (!cuda_## name) cuda_## name = cuda_ ## name ## _stub; 112 CUDALibraryProcedureList 113 #undef X 114 } 115 116 function void 117 worker_thread_sleep(GLWorkerThreadContext *ctx, BeamformerSharedMemory *sm) 118 { 119 for (;;) { 120 i32 expected = 0; 121 if (atomic_cas_u32(&ctx->sync_variable, &expected, 1) || 122 atomic_load_u32(&sm->live_imaging_parameters.active)) 123 { 124 break; 125 } 126 127 /* TODO(rnp): clean this crap up; we shouldn't need two values to communicate this */ 128 atomic_store_u32(&ctx->awake, 0); 129 os_wait_on_address(&ctx->sync_variable, 1, (u32)-1); 130 atomic_store_u32(&ctx->awake, 1); 131 } 132 } 133 134 function OS_THREAD_ENTRY_POINT_FN(compute_worker_thread_entry_point) 135 { 136 GLWorkerThreadContext *ctx = user_context; 137 138 BeamformerCtx *beamformer = (BeamformerCtx *)ctx->user_context; 139 140 for (;;) { 141 worker_thread_sleep(ctx, beamformer->shared_memory); 142 beamformer_complete_compute(beamformer, ctx->arena); 143 } 144 145 unreachable(); 146 147 return 0; 148 } 149 150 function OS_THREAD_ENTRY_POINT_FN(beamformer_upload_entry_point) 151 { 152 GLWorkerThreadContext *ctx = user_context; 153 BeamformerUploadThreadContext *up = (typeof(up))ctx->user_context; 154 155 up->rf_buffer->upload_semaphore = gpu_semaphore_create(0); 156 157 for (;;) { 158 worker_thread_sleep(ctx, up->shared_memory); 159 beamformer_rf_upload(up); 160 } 161 162 unreachable(); 163 164 return 0; 165 } 166 167 BEAMFORMER_EXPORT void * 168 beamformer_init(BeamformerInput *input) 169 { 170 Arena *memory = arena_create(.name = "Beamformer Memory"); 171 Stream error = stream_alloc(memory, MB(1)); 172 BeamformerCtx *ctx = push_struct(memory, BeamformerCtx); 173 174 for EachElement(ctx->frame_arenas, it) 175 ctx->frame_arenas[it] = arena_create(); 176 177 str8 window_title = str8("VK Beamformer"); 178 ctx->main_window = os_window_create(window_title.data, window_title.length, 1280, 840); 179 ctx->window_size = (iv2){{1280, 840}}; 180 181 ctx->arena = memory; 182 ctx->error_stream = error; 183 ctx->ui_arena = arena_create(); 184 ctx->compute_worker.arena = arena_create(); 185 ctx->upload_worker.arena = arena_create(); 186 187 #if BEAMFORMER_RENDERDOC_HOOKS 188 start_frame_capture = input->renderdoc_start_frame_capture; 189 end_frame_capture = input->renderdoc_end_frame_capture; 190 #endif 191 192 vk_load(input->vulkan_library_handle, &ctx->error_stream); 193 194 BeamformerComputeContext *cs = &ctx->compute_context; 195 196 // NOTE(rnp): allocate beamformed image ring buffer 197 { 198 u64 gpu_heap_size = gpu_info()->gpu_heap_size; 199 u64 trial_sizes[] = { 200 GB(4), 201 GB(2), 202 GB(1) + MB(512), 203 GB(1), 204 }; 205 206 u32 base_index = 0; 207 for EachElement(trial_sizes, it) { 208 if (gpu_heap_size >= 2 * trial_sizes[it]) 209 break; 210 base_index++; 211 } 212 213 for (u32 i = base_index; i < countof(trial_sizes); i++) { 214 GPUTimeline timelines[] = {GPUTimeline_Compute, GPUTimeline_Graphics}; 215 GPUBufferAllocateInfo allocate_info = { 216 .size = trial_sizes[i], 217 .flags = GPUUsageFlag_TransferDestination|GPUUsageFlag_TransferSource|GPUUsageFlag_HostRead, 218 .timeline_count = countof(timelines), 219 .timelines_used = timelines, 220 .label = str8("BeamformedData"), 221 }; 222 gpu_buffer_allocate(cs->backlog.buffer, allocate_info); 223 if (cs->backlog.buffer->size > 0) 224 break; 225 } 226 if (cs->backlog.buffer->size == 0) { 227 // NOTE(rnp): if this becomes an issue we may be able to get by in some other way 228 fatal(str8("Failed to allocate space for beamformed data\n")); 229 } 230 } 231 232 Arena *scratch = arena_create(); 233 beamformer_load_cuda_library(ctx, input->cuda_library_handle, scratch); 234 235 load_gl(&ctx->error_stream); 236 237 ctx->shared_memory = input->shared_memory; 238 ctx->shared_memory_size = input->shared_memory_size; 239 if (ctx->shared_memory_size < (i64)sizeof(*ctx->shared_memory)) 240 fatal(str8("Get more ram lol\n")); 241 zero_struct(ctx->shared_memory); 242 243 ctx->shared_memory->version = BEAMFORMER_SHARED_MEMORY_VERSION; 244 ctx->shared_memory->reserved_parameter_blocks = 1; 245 { 246 BeamformerParameterBlock *pb = beamformer_parameter_block(ctx->shared_memory, 0); 247 zero_struct(pb); 248 } 249 250 ctx->shared_memory->beamformed_frame_buffer_size = cs->backlog.buffer->size; 251 252 // TODO(rnp): dynamic rf data buffer slot usage 253 // NOTE(rnp): will be same as the max size we were able to get for the frame buffer 254 ctx->shared_memory->capabilities.max_rf_data_size = cs->backlog.buffer->size 255 / BeamformerMaxRawDataFramesInFlight; 256 257 ctx->shared_memory->capabilities.cuda = cuda_supported(); 258 // TODO(rnp): re-enable hilbert support, with and without cuda 259 ctx->shared_memory->capabilities.hilbert = 0; 260 261 /* TODO(rnp): I'm not sure if its a good idea to pre-reserve a bunch of semaphores 262 * on w32 but thats what we are doing for now */ 263 #if OS_WINDOWS 264 { 265 Stream sb = arena_stream(memory); 266 stream_append(&sb, input->shared_memory_name, input->shared_memory_name_length); 267 stream_append_str8(&sb, str8("_lock_")); 268 i32 start_index = sb.widx; 269 for EachElement(os_w32_shared_memory_semaphores, it) { 270 stream_reset(&sb, start_index); 271 stream_append_u64(&sb, it); 272 stream_append_byte(&sb, 0); 273 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 274 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 275 fatal(beamformer_info("init: failed to create w32 shared memory semaphore\n")); 276 277 /* NOTE(rnp): hacky garbage because CreateSemaphore will just open an existing 278 * semaphore without any indication. Sometimes the other side of the shared memory 279 * will provide incorrect parameters or will otherwise fail and its faster to 280 * restart this program than to get that application to release the semaphores */ 281 /* TODO(rnp): figure out something more robust */ 282 os_w32_semaphore_release(os_w32_shared_memory_semaphores[it], 1); 283 } 284 } 285 #endif 286 287 GLWorkerThreadContext *worker = &ctx->compute_worker; 288 /* TODO(rnp): we should lock this down after we have something working */ 289 worker->user_context = (iptr)ctx; 290 worker->handle = os_create_thread("[compute]", worker, compute_worker_thread_entry_point); 291 292 GLWorkerThreadContext *upload = &ctx->upload_worker; 293 BeamformerUploadThreadContext *upctx = push_struct(memory, typeof(*upctx)); 294 upload->user_context = (iptr)upctx; 295 upctx->rf_buffer = &cs->rf_buffer; 296 upctx->shared_memory = ctx->shared_memory; 297 upctx->shared_memory_size = ctx->shared_memory_size; 298 upctx->compute_timing_table = ctx->compute_timing_table; 299 upctx->compute_worker_sync = &ctx->compute_worker.sync_variable; 300 upload->handle = os_create_thread("[upload]", upload, beamformer_upload_entry_point); 301 302 /* NOTE: set up OpenGL debug logging */ 303 Stream *gl_error_stream = push_struct(memory, Stream); 304 *gl_error_stream = stream_alloc(memory, 1024); 305 glDebugMessageCallback(gl_debug_logger, gl_error_stream); 306 #ifdef BEAMFORMER_DEBUG 307 glEnable(GL_DEBUG_OUTPUT); 308 #endif 309 310 if (!BakeShaders) 311 { 312 for EachElement(beamformer_reloadable_compute_shader_info_indices, it) { 313 i32 index = beamformer_reloadable_compute_shader_info_indices[it]; 314 315 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 316 beamformer_reloadable_shader_files[index][0]); 317 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 318 frc->kind = BeamformerFileReloadKind_ComputeShader; 319 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 320 os_add_file_watch((char *)file.data, file.length, frc); 321 } 322 323 for EachElement(beamformer_reloadable_compute_helpers_shader_info_indices, it) { 324 i32 index = beamformer_reloadable_compute_helpers_shader_info_indices[it]; 325 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 326 beamformer_reloadable_shader_files[index][0]); 327 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 328 frc->kind = BeamformerFileReloadKind_ComputeShader; 329 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 330 os_add_file_watch((char *)file.data, file.length, frc); 331 } 332 } 333 334 arena_destroy(scratch); 335 336 ctx->state = BeamformerState_Running; 337 338 return ctx; 339 } 340 341 BEAMFORMER_EXPORT void 342 beamformer_terminate(void *memory, BeamformerInput *input) 343 { 344 /* NOTE(rnp): work around pebkac when the beamformer is closed while we are doing live 345 * imaging. if the verasonics is blocked in an external function (calling the library 346 * to start compute) it is impossible for us to get it to properly shut down which 347 * will sometimes result in us needing to power cycle the system. set the shared memory 348 * into an error state and release dispatch lock so that future calls will error instead 349 * of blocking. 350 */ 351 BeamformerCtx *ctx = memory; 352 BeamformerSharedMemory *sm = input->shared_memory; 353 if (ctx->state != BeamformerState_Terminated) { 354 if (sm) { 355 BeamformerSharedMemoryLockKind lock = BeamformerSharedMemoryLockKind_DispatchCompute; 356 atomic_store_u32(&sm->invalid, 1); 357 atomic_store_u32(&sm->external_work_queue.ridx, sm->external_work_queue.widx); 358 DEBUG_DECL(if (sm->locks[lock])) { 359 beamformer_shared_memory_release_lock(sm, (i32)lock); 360 } 361 362 atomic_or_u32(&sm->live_imaging_dirty_flags, BeamformerLiveImagingDirtyFlags_StopImaging); 363 } 364 365 beamformer_debug_ui_deinit(ctx); 366 367 ctx->state = BeamformerState_Terminated; 368 } 369 } 370 371 BEAMFORMER_EXPORT u32 372 beamformer_should_close(void *memory, BeamformerInput *input) 373 { 374 BeamformerCtx *ctx = memory; 375 if (ctx->state == BeamformerState_ShouldClose) 376 beamformer_terminate(memory, input); 377 return ctx->state == BeamformerState_Terminated; 378 }