beamformer.c (13339B)
1 /* See LICENSE for license details. */ 2 3 #include "beamformer_internal.h" 4 5 /* NOTE(rnp): magic variables to force discrete GPU usage on laptops with multiple devices */ 6 EXPORT i32 NvOptimusEnablement = 1; 7 EXPORT i32 AmdPowerXpressRequestHighPerformance = 1; 8 9 #if !BEAMFORMER_DEBUG 10 #include "beamformer_core.c" 11 #else 12 13 typedef void beamformer_frame_step_fn(void *, BeamformerInput *); 14 15 #define BEAMFORMER_DEBUG_ENTRY_POINTS \ 16 X(beamformer_debug_ui_deinit) \ 17 X(beamformer_complete_compute) \ 18 X(beamformer_frame_step) \ 19 X(beamformer_rf_upload) \ 20 21 #define X(name) global name ##_fn *name; 22 BEAMFORMER_DEBUG_ENTRY_POINTS 23 #undef X 24 25 BEAMFORMER_EXPORT void 26 beamformer_debug_hot_release(void *memory, BeamformerInput *input) 27 { 28 BeamformerCtx *ctx = memory; 29 // TODO(rnp): this will deadlock if live imaging is active 30 /* NOTE(rnp): spin until compute thread finishes its work (we will probably 31 * never reload while compute is in progress but just incase). */ 32 spin_while_true(atomic_load_u32(&ctx->upload_worker.awake)); 33 spin_while_true(atomic_load_u32(&ctx->compute_worker.awake)); 34 } 35 36 BEAMFORMER_EXPORT void 37 beamformer_debug_hot_reload(OSLibrary library) 38 { 39 #define X(name) name = os_lookup_symbol(library, #name); 40 BEAMFORMER_DEBUG_ENTRY_POINTS 41 #undef X 42 43 str8 info = beamformer_info("reloaded main executable"); 44 os_console_log(info.data, info.length); 45 } 46 47 #endif /* BEAMFORMER_DEBUG */ 48 49 function no_return void 50 fatal(str8 message) 51 { 52 os_fatal(message.data, message.length); 53 unreachable(); 54 } 55 56 #include "vulkan.c" 57 58 // TODO(rnp): this doesn't belong here, but will be removed 59 // once vulkan migration is complete 60 void * glfwGetProcAddress(char *); 61 62 function void 63 gl_debug_logger(u32 src, u32 type, u32 id, u32 lvl, i32 len, const char *msg, const void *userctx) 64 { 65 Stream *e = (Stream *)userctx; 66 stream_append_str8s(e, str8("[OpenGL] "), (str8){.length = len, .data = (u8 *)msg}, str8("\n")); 67 os_console_log(e->data, e->widx); 68 stream_reset(e, 0); 69 } 70 71 function void 72 load_gl(Stream *err) 73 { 74 #define X(name, ret, params) name = (name##_fn *)glfwGetProcAddress(#name); 75 OGLProcedureList 76 OGLRequiredExtensionProcedureList 77 #undef X 78 79 stream_reset(err, 0); 80 #define X(name, ret, params) if (!name) stream_append_str8(err, str8("missing required GL function: " #name "\n")); 81 OGLProcedureList 82 OGLRequiredExtensionProcedureListBase 83 #if OS_WINDOWS 84 OGLRequiredExtensionProcedureListW32 85 #else 86 OGLRequiredExtensionProcedureListLinux 87 #endif 88 #undef X 89 90 if (err->widx) fatal(stream_to_str8(err)); 91 } 92 93 function void 94 beamformer_load_cuda_library(BeamformerCtx *ctx, OSLibrary cuda, Arena *scratch) 95 { 96 /* TODO(rnp): (25.10.30) registering the rf buffer with CUDA is currently 97 * causing a major performance regression. for now we are disabling its use 98 * altogether. it will be reenabled once the issue can be fixed */ 99 b32 result = 0 && gpu_info()->vendor == GPUVendor_NVIDIA && ValidHandle(cuda); 100 if (result) { 101 Stream err = arena_stream(scratch); 102 103 stream_append_str8(&err, beamformer_info("loading CUDA library functions")); 104 #define X(name, symname) cuda_## name = os_lookup_symbol(cuda, symname); 105 CUDALibraryProcedureList 106 #undef X 107 108 os_console_log(err.data, err.widx); 109 } 110 111 #define X(name, symname) if (!cuda_## name) cuda_## name = cuda_ ## name ## _stub; 112 CUDALibraryProcedureList 113 #undef X 114 } 115 116 function void 117 worker_thread_sleep(GLWorkerThreadContext *ctx, BeamformerSharedMemory *sm) 118 { 119 for (;;) { 120 i32 expected = 0; 121 if (atomic_cas_u32(&ctx->sync_variable, &expected, 1) || 122 atomic_load_u32(&sm->live_imaging_parameters.active)) 123 { 124 break; 125 } 126 127 /* TODO(rnp): clean this crap up; we shouldn't need two values to communicate this */ 128 atomic_store_u32(&ctx->awake, 0); 129 os_wait_on_address(&ctx->sync_variable, 1, (u32)-1); 130 atomic_store_u32(&ctx->awake, 1); 131 } 132 } 133 134 function OS_THREAD_ENTRY_POINT_FN(compute_worker_thread_entry_point) 135 { 136 GLWorkerThreadContext *ctx = user_context; 137 138 BeamformerCtx *beamformer = (BeamformerCtx *)ctx->user_context; 139 140 for (;;) { 141 worker_thread_sleep(ctx, beamformer->shared_memory); 142 beamformer_complete_compute(beamformer, ctx->arena); 143 } 144 145 unreachable(); 146 147 return 0; 148 } 149 150 function OS_THREAD_ENTRY_POINT_FN(beamformer_upload_entry_point) 151 { 152 GLWorkerThreadContext *ctx = user_context; 153 BeamformerUploadThreadContext *up = (typeof(up))ctx->user_context; 154 155 up->rf_buffer->upload_semaphore = gpu_semaphore_create(0); 156 157 for (;;) { 158 worker_thread_sleep(ctx, up->shared_memory); 159 beamformer_rf_upload(up); 160 } 161 162 unreachable(); 163 164 return 0; 165 } 166 167 BEAMFORMER_EXPORT void * 168 beamformer_init(BeamformerInput *input) 169 { 170 Arena *memory = arena_create(.name = "Beamformer Memory"); 171 Stream error = stream_alloc(memory, MB(1)); 172 BeamformerCtx *ctx = push_struct(memory, BeamformerCtx); 173 174 for EachElement(ctx->frame_arenas, it) 175 ctx->frame_arenas[it] = arena_create(); 176 177 str8 window_title = str8("VK Beamformer"); 178 ctx->main_window = os_window_create(window_title.data, window_title.length, 1280, 840); 179 ctx->window_size = (iv2){{1280, 840}}; 180 181 ctx->arena = memory; 182 ctx->error_stream = error; 183 ctx->ui_arena = arena_create(); 184 ctx->compute_worker.arena = arena_create(); 185 ctx->upload_worker.arena = arena_create(); 186 187 #if BEAMFORMER_RENDERDOC_HOOKS 188 start_frame_capture = input->renderdoc_start_frame_capture; 189 end_frame_capture = input->renderdoc_end_frame_capture; 190 #endif 191 192 vk_load(input->vulkan_library_handle, &ctx->error_stream); 193 194 BeamformerComputeContext *cs = &ctx->compute_context; 195 196 // NOTE(rnp): allocate beamformed image ring buffer 197 { 198 u64 gpu_heap_size = gpu_info()->gpu_heap_size; 199 u64 trial_sizes[] = { 200 GB(4), 201 GB(2), 202 GB(1) + MB(512), 203 GB(1), 204 }; 205 206 u32 base_index = 0; 207 for EachElement(trial_sizes, it) { 208 if (gpu_heap_size >= 2 * trial_sizes[it]) 209 break; 210 base_index++; 211 } 212 213 for (u32 i = base_index; i < countof(trial_sizes); i++) { 214 GPUTimeline timelines[] = {GPUTimeline_Compute, GPUTimeline_Graphics}; 215 GPUBufferAllocateInfo allocate_info = { 216 .size = trial_sizes[i], 217 .flags = GPUUsageFlag_TransferDestination|GPUUsageFlag_TransferSource|GPUUsageFlag_HostRead, 218 .timeline_count = countof(timelines), 219 .timelines_used = timelines, 220 .label = str8("BeamformedData"), 221 }; 222 gpu_buffer_allocate(cs->backlog.buffer, allocate_info); 223 if (cs->backlog.buffer->size > 0) 224 break; 225 } 226 if (cs->backlog.buffer->size == 0) { 227 // NOTE(rnp): if this becomes an issue we may be able to get by in some other way 228 fatal(str8("Failed to allocate space for beamformed data\n")); 229 } 230 } 231 232 Arena *scratch = arena_create(); 233 beamformer_load_cuda_library(ctx, input->cuda_library_handle, scratch); 234 235 load_gl(&ctx->error_stream); 236 237 //////////////////////////// 238 // NOTE(rnp): shared memory setup 239 { 240 BeamformerSharedMemory *sm = input->shared_memory; 241 ctx->shared_memory = sm; 242 ctx->shared_memory_size = input->shared_memory_size; 243 if (ctx->shared_memory_size < (i64)sizeof(*ctx->shared_memory)) 244 fatal(str8("Get more ram lol\n")); 245 zero_struct(sm); 246 247 sm->version = BEAMFORMER_SHARED_MEMORY_VERSION; 248 sm->reserved_parameter_blocks = 1; 249 { 250 BeamformerParameterBlock *pb = beamformer_parameter_block(sm, 0); 251 zero_struct(pb); 252 } 253 254 sm->beamformed_frame_buffer_size = cs->backlog.buffer->size; 255 256 // TODO(rnp): dynamic rf data buffer slot usage 257 // NOTE(rnp): will be same as the max size we were able to get for the frame buffer 258 sm->capabilities.max_rf_data_size = cs->backlog.buffer->size / BeamformerMaxRawDataFramesInFlight; 259 260 sm->capabilities.cuda = cuda_supported(); 261 // TODO(rnp): re-enable hilbert support, with and without cuda 262 sm->capabilities.hilbert = 0; 263 264 sm->transient.offset = AlignUpPowerOfTwo(sizeof(*sm) + BeamformerMaxParameterBlocks * sizeof(BeamformerParameterBlock), 265 BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 266 sm->transient.page_count = DivRoundUp(ctx->shared_memory_size - sm->transient.offset, 267 BEAMFORMER_SHARED_MEMORY_TRANSIENT_PAGE_SIZE); 268 sm->transient.first_free_page = 0; 269 BeamformerFreePageRun *pr = (void *)((u8 *)sm + sm->transient.offset); 270 pr->page_count = sm->transient.page_count; 271 pr->next_page = pr->prev_page = -1u; 272 } 273 274 /* TODO(rnp): I'm not sure if its a good idea to pre-reserve a bunch of semaphores 275 * on w32 but thats what we are doing for now */ 276 #if OS_WINDOWS 277 { 278 Stream sb = arena_stream(memory); 279 stream_append(&sb, input->shared_memory_name, input->shared_memory_name_length); 280 stream_append_str8(&sb, str8("_lock_")); 281 i32 start_index = sb.widx; 282 for EachElement(os_w32_shared_memory_semaphores, it) { 283 stream_reset(&sb, start_index); 284 stream_append_u64(&sb, it); 285 stream_append_byte(&sb, 0); 286 os_w32_shared_memory_semaphores[it] = os_w32_create_semaphore((c8 *)sb.data, 1, 1); 287 if InvalidHandle(os_w32_shared_memory_semaphores[it]) 288 fatal(beamformer_info("init: failed to create w32 shared memory semaphore\n")); 289 290 /* NOTE(rnp): hacky garbage because CreateSemaphore will just open an existing 291 * semaphore without any indication. Sometimes the other side of the shared memory 292 * will provide incorrect parameters or will otherwise fail and its faster to 293 * restart this program than to get that application to release the semaphores */ 294 /* TODO(rnp): figure out something more robust */ 295 os_w32_semaphore_release(os_w32_shared_memory_semaphores[it], 1); 296 } 297 } 298 #endif 299 300 GLWorkerThreadContext *worker = &ctx->compute_worker; 301 /* TODO(rnp): we should lock this down after we have something working */ 302 worker->user_context = (iptr)ctx; 303 worker->handle = os_create_thread("[compute]", worker, compute_worker_thread_entry_point); 304 305 GLWorkerThreadContext *upload = &ctx->upload_worker; 306 BeamformerUploadThreadContext *upctx = push_struct(memory, typeof(*upctx)); 307 upload->user_context = (iptr)upctx; 308 upctx->rf_buffer = &cs->rf_buffer; 309 upctx->shared_memory = ctx->shared_memory; 310 upctx->shared_memory_size = ctx->shared_memory_size; 311 upctx->compute_timing_table = ctx->compute_timing_table; 312 upctx->compute_worker_sync = &ctx->compute_worker.sync_variable; 313 upload->handle = os_create_thread("[upload]", upload, beamformer_upload_entry_point); 314 315 /* NOTE: set up OpenGL debug logging */ 316 Stream *gl_error_stream = push_struct(memory, Stream); 317 *gl_error_stream = stream_alloc(memory, 1024); 318 glDebugMessageCallback(gl_debug_logger, gl_error_stream); 319 #ifdef BEAMFORMER_DEBUG 320 glEnable(GL_DEBUG_OUTPUT); 321 #endif 322 323 if (!BakeShaders) 324 { 325 for EachElement(beamformer_reloadable_compute_shader_info_indices, it) { 326 i32 index = beamformer_reloadable_compute_shader_info_indices[it]; 327 328 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 329 beamformer_reloadable_shader_files[index][0]); 330 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 331 frc->kind = BeamformerFileReloadKind_ComputeShader; 332 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 333 os_add_file_watch((char *)file.data, file.length, frc); 334 } 335 336 for EachElement(beamformer_reloadable_compute_helpers_shader_info_indices, it) { 337 i32 index = beamformer_reloadable_compute_helpers_shader_info_indices[it]; 338 str8 file = push_str8_from_parts(scratch, os_path_separator(), str8("shaders"), 339 beamformer_reloadable_shader_files[index][0]); 340 BeamformerFileReloadContext *frc = push_struct(memory, typeof(*frc)); 341 frc->kind = BeamformerFileReloadKind_ComputeShader; 342 frc->shader_reload.shader = beamformer_reloadable_shader_kinds[index]; 343 os_add_file_watch((char *)file.data, file.length, frc); 344 } 345 } 346 347 arena_destroy(scratch); 348 349 ctx->state = BeamformerState_Running; 350 351 return ctx; 352 } 353 354 BEAMFORMER_EXPORT void 355 beamformer_terminate(void *memory, BeamformerInput *input) 356 { 357 /* NOTE(rnp): work around pebkac when the beamformer is closed while we are doing live 358 * imaging. if the verasonics is blocked in an external function (calling the library 359 * to start compute) it is impossible for us to get it to properly shut down which 360 * will sometimes result in us needing to power cycle the system. set the shared memory 361 * into an error state and release dispatch lock so that future calls will error instead 362 * of blocking. 363 */ 364 BeamformerCtx *ctx = memory; 365 BeamformerSharedMemory *sm = input->shared_memory; 366 if (ctx->state != BeamformerState_Terminated) { 367 if (sm) { 368 BeamformerSharedMemoryLockKind lock = BeamformerSharedMemoryLockKind_DispatchCompute; 369 atomic_store_u32(&sm->invalid, 1); 370 atomic_store_u32(&sm->external_work_queue.ridx, sm->external_work_queue.widx); 371 DEBUG_DECL(if (sm->locks[lock])) { 372 beamformer_shared_memory_release_lock(sm, (i32)lock); 373 } 374 375 atomic_or_u32(&sm->live_imaging_dirty_flags, BeamformerLiveImagingDirtyFlags_StopImaging); 376 } 377 378 beamformer_debug_ui_deinit(ctx); 379 380 ctx->state = BeamformerState_Terminated; 381 } 382 } 383 384 BEAMFORMER_EXPORT u32 385 beamformer_should_close(void *memory, BeamformerInput *input) 386 { 387 BeamformerCtx *ctx = memory; 388 if (ctx->state == BeamformerState_ShouldClose) 389 beamformer_terminate(memory, input); 390 return ctx->state == BeamformerState_Terminated; 391 }