Commit: e073c29d2fbc1e71ed3bc835d5b25c4f5b6d4405
Parent: 5095bc2ebafc453774d521ed1eee2f08c38bd51d
Author: Randy Palamar
Date: Tue, 31 Dec 2024 23:51:36 -0700
portage: add renderdoc patch for properly attaching to process with musl
see: https://github.com/baldurk/renderdoc/issues/3228
Its probably better write a more correct patch for this at some point
Diffstat:
2 files changed, 581 insertions(+), 1 deletion(-)
diff --git a/.config/sys/etc/portage/package.accept_keywords/set b/.config/sys/etc/portage/package.accept_keywords/set
@@ -32,7 +32,6 @@ sys-process/nvtop ~amd64 ~arm64
# No stable version
~x11-misc/xidle-26052015 ~amd64 ~arm64
-media-gfx/renderdoc ~amd64 ~arm64
media-video/obs-studio ~amd64 ~arm64
www-client/firefox ~amd64 ~arm64
diff --git a/.config/sys/etc/portage/patches/media-gfx/renderdoc/musl_program_injection.patch b/.config/sys/etc/portage/patches/media-gfx/renderdoc/musl_program_injection.patch
@@ -0,0 +1,581 @@
+From 41ac532478fe95013ec2a2570aa49a7e7c787920 Mon Sep 17 00:00:00 2001
+From: Jules Maselbas <jmaselbas@zdiv.net>
+Date: Mon, 22 Jan 2024 17:55:21 +0100
+Subject: [PATCH 2/2] Rework injection and synchronisaion on linux
+
+for the record, modification to "fix" https://github.com/baldurk/renderdoc/issues/3228
+---
+ renderdoc/os/posix/linux/linux_hook.cpp | 2 -
+ renderdoc/os/posix/linux/linux_network.cpp | 5 +-
+ renderdoc/os/posix/linux/linux_process.cpp | 500 +--------------------
+ renderdoc/os/posix/posix_process.cpp | 3 -
+ 4 files changed, 12 insertions(+), 498 deletions(-)
+
+diff --git a/renderdoc/os/posix/linux/linux_hook.cpp b/renderdoc/os/posix/linux/linux_hook.cpp
+index 7138904d87..1a8ba65cb9 100644
+--- a/renderdoc/os/posix/linux/linux_hook.cpp
++++ b/renderdoc/os/posix/linux/linux_hook.cpp
+@@ -313,8 +313,6 @@ __attribute__((visibility("default"))) pid_t fork()
+ {
+ if(Linux_Debug_PtraceLogging())
+ RDCLOG("hooked fork() in child %d", getpid());
+-
+- StopAtMainInChild();
+ }
+ else if(ret > 0)
+ {
+diff --git a/renderdoc/os/posix/linux/linux_network.cpp b/renderdoc/os/posix/linux/linux_network.cpp
+index 4b67d3d18e..ba333db13f 100644
+--- a/renderdoc/os/posix/linux/linux_network.cpp
++++ b/renderdoc/os/posix/linux/linux_network.cpp
+@@ -24,6 +24,7 @@
+
+ #include "os/os_specific.h"
+ #include "os/posix/posix_network.h"
++void StopAtMainInChild();
+
+ namespace Network
+ {
+@@ -39,6 +40,8 @@ uint32_t Socket::GetRemoteIP() const
+
+ Socket *CreateServerSocket(const rdcstr &bindaddr, uint16_t port, int queuesize)
+ {
+- return CreateTCPServerSocket(bindaddr, port, queuesize);
++ Socket *s = CreateTCPServerSocket(bindaddr, port, queuesize);
++ StopAtMainInChild();
++ return s;
+ }
+ };
+diff --git a/renderdoc/os/posix/linux/linux_process.cpp b/renderdoc/os/posix/linux/linux_process.cpp
+index 4130e9e8dc..0a5831b3a0 100644
+--- a/renderdoc/os/posix/linux/linux_process.cpp
++++ b/renderdoc/os/posix/linux/linux_process.cpp
+@@ -164,506 +164,22 @@ int GetIdentPort(pid_t childPid)
+ return ret;
+ }
+
+-static bool ptrace_scope_ok()
+-{
+- if(!Linux_PtraceChildProcesses())
+- return false;
+-
+- rdcstr contents;
+- FileIO::ReadAll("/proc/sys/kernel/yama/ptrace_scope", contents);
+- contents.trim();
+- if(!contents.empty())
+- {
+- int ptrace_scope = atoi(contents.c_str());
+- if(ptrace_scope > 1)
+- {
+- if(RenderDoc::Inst().IsReplayApp())
+- {
+- static bool warned = false;
+- if(!warned)
+- {
+- warned = true;
+- RDCWARN(
+- "ptrace_scope value %d means ptrace can't be used to pause child processes while "
+- "attaching.",
+- ptrace_scope);
+- }
+- }
+- return false;
+- }
+- }
+-
+- return true;
+-}
+-
+-static uint64_t get_nanotime()
+-{
+- timespec ts;
+- clock_gettime(CLOCK_MONOTONIC, &ts);
+- uint64_t ret = uint64_t(ts.tv_sec) * 1000000000ULL + uint32_t(ts.tv_nsec & 0xffffffff);
+- return ret;
+-}
+-
+-#if defined(__arm__)
+-
+-// for some reason arm doesn't have the same struct name. Sigh :(
+-#define user_regs_struct user_regs
+-
+-#define INST_PTR_REG ARM_pc
+-
+-#define BREAK_INST 0xe7f001f0ULL
+-#define BREAK_INST_BYTES_SIZE 4
+-// on ARM seemingly the instruction isn't actually considered executed, so we don't have to modify
+-// the instruction pointer at all.
+-#define BREAK_INST_INST_PTR_ADJUST 0
+-
+-#elif defined(__aarch64__)
+-
+-#define INST_PTR_REG pc
+-
+-#define BREAK_INST 0xd4200000ULL
+-#define BREAK_INST_BYTES_SIZE 4
+-// on ARM seemingly the instruction isn't actually considered executed, so we don't have to modify
+-// the instruction pointer at all.
+-#define BREAK_INST_INST_PTR_ADJUST 0
+-
+-#elif defined(__riscv)
+-
+-#define INST_PTR_REG pc
+-
+-// ebreak
+-#define BREAK_INST 0x00100073ULL
+-#define BREAK_INST_BYTES_SIZE 4
+-#define BREAK_INST_INST_PTR_ADJUST 4
+-
+-#else
+-
+-#define BREAK_INST 0xccULL
+-#define BREAK_INST_BYTES_SIZE 1
+-// step back over the instruction
+-#define BREAK_INST_INST_PTR_ADJUST 1
+-
+-#if ENABLED(RDOC_X64)
+-#define INST_PTR_REG rip
+-#else
+-#define INST_PTR_REG eip
+-#endif
+-
+-#endif
+-
+-static uint64_t get_child_ip(pid_t childPid)
+-{
+- user_regs_struct regs = {};
+-
+- iovec regs_iovec = {®s, sizeof(regs)};
+- long ptraceRet = ptrace(PTRACE_GETREGSET, childPid, (void *)NT_PRSTATUS, ®s_iovec);
+- if(ptraceRet == 0)
+- return uint64_t(regs.INST_PTR_REG);
+-
+- return 0;
+-}
+-
+-static bool wait_traced_child(pid_t childPid, uint32_t timeoutMS, int &status)
+-{
+- // spin waiting for the traced child, with a 100ms timeout
+- status = 0;
+- uint64_t start_nano = get_nanotime();
+- uint64_t end_nano = 0;
+- int ret = 0;
+-
+- const uint64_t timeoutNanoseconds = uint64_t(timeoutMS) * 1000 * 1000;
+-
+- while((ret = waitpid(childPid, &status, WNOHANG)) == 0)
+- {
+- status = 0;
+-
+- // if we're in a capturing process then the process itself might have done waitpid(-1) and
+- // swallowed the wait for our child. So as an alternative we check to see if we can query the
+- // instruction pointer, which is only possible if the child is stopped.
+- uint64_t ip = get_child_ip(childPid);
+- if(ip != 0)
+- {
+- // do waitpid again in case we raced and the child stopped in between the call to waitpid and
+- // get_child_ip.
+- ret = waitpid(childPid, &status, WNOHANG);
+-
+- // if it still didn't succeed, set status to 0 so we know we're earlying out and don't check
+- // the status codes.
+- if(ret == 0)
+- status = 0;
+- return true;
+- }
+-
+- usleep(10);
+-
+- // check the timeout
+- end_nano = get_nanotime();
+- if(end_nano - start_nano > timeoutNanoseconds)
+- break;
+- }
+-
+- return WIFSTOPPED(status);
+-}
+-
+ bool StopChildAtMain(pid_t childPid, bool *exitWithNoExec)
+ {
+- // don't do this unless the ptrace scope is OK.
+- if(!ptrace_scope_ok())
+- return false;
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Stopping child PID %u at main", childPid);
+-
+- int childStatus = 0;
+-
+- // we have a low timeout for this stop since it should happen almost immediately (right after the
+- // fork). If it didn't then we want to fail relatively fast.
+- if(!wait_traced_child(childPid, 100, childStatus))
+- {
+- RDCERR("Didn't get initial stop from child PID %u", childPid);
+- return false;
+- }
+-
+- if(childStatus > 0 && WSTOPSIG(childStatus) != SIGSTOP)
+- {
+- RDCERR("Initial signal from child PID %u was %x, expected %x", childPid, WSTOPSIG(childStatus),
+- SIGSTOP);
+- return false;
+- }
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Child PID %u is stopped in StopAtMainInChild()", childPid);
+-
+- int64_t ptraceRet = 0;
+-
+- // continue until exec
+- ptraceRet = ptrace(PTRACE_SETOPTIONS, childPid, NULL, PTRACE_O_TRACEEXEC | PTRACE_O_TRACEEXIT);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Child PID %u configured to trace exec(). Continuing child", childPid);
+-
+- // continue
+- ptraceRet = ptrace(PTRACE_CONT, childPid, NULL, NULL);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- // we're not under control of when the application calls exec() after fork() in the case of child
+- // processes, so be a little more generous with the timeout
+- if(!wait_traced_child(childPid, 250, childStatus))
+- {
+- RDCERR("Didn't get to execve in child PID %u", childPid);
+- return false;
+- }
+-
+- int statusResult = childStatus >> 8;
+-
+- if(childStatus > 0 &&
+- (statusResult == SIGCHLD || statusResult == (SIGTRAP | (PTRACE_EVENT_EXIT << 8))))
+- {
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Child PID %u exited while waiting for exec() 0x%x", childPid, childStatus);
+- if(exitWithNoExec)
+- *exitWithNoExec = true;
+-
+- if(statusResult == SIGCHLD)
+- ptrace(PTRACE_DETACH, childPid, NULL, SIGCHLD);
+- else
+- ptrace(PTRACE_DETACH, childPid, NULL, NULL);
+- return false;
+- }
+-
+- if(childStatus > 0 && statusResult != (SIGTRAP | (PTRACE_EVENT_EXEC << 8)))
+- {
+- RDCERR("Exec wait event from child PID %u was status 0x%x, expected 0x%x", childPid,
+- statusResult, (SIGTRAP | (PTRACE_EVENT_EXEC << 8)));
+-
+- return false;
+- }
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Child PID %u is stopped at execve() 0x%x", childPid, childStatus);
+-
+- rdcstr exepath;
+- long baseVirtualPointer = 0;
+- uint32_t sectionOffset = 0;
+-
+- rdcstr mapsName = StringFormat::Fmt("/proc/%u/maps", childPid);
+-
+- FILE *maps = FileIO::fopen(mapsName, FileIO::ReadText);
+-
+- if(!maps)
+- {
+- RDCERR("Couldn't open %s", mapsName.c_str());
+- return false;
+- }
+-
+- while(!feof(maps))
+- {
+- char line[512] = {0};
+- if(fgets(line, 511, maps))
+- {
+- if(strstr(line, "r-xp"))
+- {
+- RDCCOMPILE_ASSERT(sizeof(long) == sizeof(void *), "Expected long to be pointer sized");
+- int pathOffset = 0;
+- int num = sscanf(line, "%lx-%*x r-xp %x %*x:%*x %*u %n", &baseVirtualPointer,
+- §ionOffset, &pathOffset);
+-
+- if(num != 2 || pathOffset == 0)
+- {
+- RDCERR("Couldn't parse first executable mapping '%s'", rdcstr(line).trimmed().c_str());
+- return false;
+- }
+-
+- exepath = line + pathOffset;
+- exepath.trim();
+- break;
+- }
+- }
+- }
+-
+- if(baseVirtualPointer == 0)
+- {
+- RDCERR("Couldn't find executable mapping in maps file");
+- return false;
+- }
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Child PID %u has exepath %s basePointer 0x%llx and sectionOffset 0x%x", childPid,
+- exepath.c_str(), (uint64_t)baseVirtualPointer, (uint32_t)sectionOffset);
+-
+- FileIO::fclose(maps);
+-
+- FILE *elf = FileIO::fopen(exepath, FileIO::ReadText);
+-
+- if(!elf)
+- {
+- RDCERR("Couldn't open %s to parse ELF header", exepath.c_str());
+- return false;
+- }
+-
+- Elf64_Ehdr elf_header;
+- size_t read = FileIO::fread(&elf_header, sizeof(elf_header), 1, elf);
+-
+- if(read != 1)
+- {
+- FileIO::fclose(elf);
+- RDCERR("Couldn't read ELF header from %s", exepath.c_str());
+- return false;
+- }
+-
+- size_t entryVirtual = (size_t)elf_header.e_entry;
+- // if the section doesn't shift between file offset and virtual address this will be the same
+- size_t entryFileOffset = entryVirtual;
+-
+- if(elf_header.e_shoff)
+- {
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("exepath %s contains sections, rebasing to correct section", exepath.c_str());
+-
+- FileIO::fseek64(elf, elf_header.e_shoff, SEEK_SET);
+-
+- RDCASSERTEQUAL(elf_header.e_shentsize, sizeof(Elf64_Shdr));
+-
+- for(Elf64_Half s = 0; s < elf_header.e_shnum; s++)
+- {
+- Elf64_Shdr section_header;
+- read = FileIO::fread(§ion_header, sizeof(section_header), 1, elf);
+-
+- if(read != 1)
+- {
+- FileIO::fclose(elf);
+- RDCERR("Couldn't read section header from %s", exepath.c_str());
+- return false;
+- }
+-
+- if(section_header.sh_addr <= entryVirtual &&
+- entryVirtual < section_header.sh_addr + section_header.sh_size)
+- {
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG(
+- "Found section in %s from 0x%llx - 0x%llx at offset 0x%llx containing entry 0x%llx.",
+- exepath.c_str(), (uint64_t)section_header.sh_addr,
+- uint64_t(section_header.sh_addr + section_header.sh_size),
+- (uint64_t)section_header.sh_offset, (uint64_t)entryVirtual);
+-
+- entryFileOffset =
+- (entryVirtual - (size_t)section_header.sh_addr) + (size_t)section_header.sh_offset;
+-
+- break;
+- }
+- }
+- }
+-
+- FileIO::fclose(elf);
+-
+- void *entry = (void *)(baseVirtualPointer + entryFileOffset - sectionOffset);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("child process %u executable %s has entry %p at 0x%llx + (0x%llx - 0x%x)", childPid,
+- exepath.c_str(), entry, (uint64_t)baseVirtualPointer, (uint64_t)entryFileOffset,
+- (uint32_t)sectionOffset);
+-
+- // this reads a 'word' and returns as long, upcast (if needed) to uint64_t
+- uint64_t origEntryWord = (uint64_t)ptrace(PTRACE_PEEKTEXT, childPid, entry, 0);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Read word %llx from %p in child process %u running executable %s",
+- (uint64_t)origEntryWord, entry, childPid, exepath.c_str());
+-
+- uint64_t breakpointWord =
+- (origEntryWord & (0xffffffffffffffffULL << (BREAK_INST_BYTES_SIZE * 8))) | BREAK_INST;
+- // downcast back to long, if that means truncating
+- ptraceRet = ptrace(PTRACE_POKETEXT, childPid, entry, (long)breakpointWord);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Changed word to %llx and re-poked in process %u. Continuing child",
+- (uint64_t)breakpointWord, childPid);
+-
+- // continue
+- ptraceRet = ptrace(PTRACE_CONT, childPid, NULL, NULL);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- // it could take a long time to hit main so we have a large timeout here
+- if(!wait_traced_child(childPid, 2000, childStatus))
+- {
+- RDCERR("Didn't hit breakpoint in PID %u (%x)", childPid, childStatus);
+- return false;
+- }
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Process %u hit entry point", childPid);
+-
+- // we're now at main! now just need to clean up after ourselves
+-
+- user_regs_struct regs = {};
+-
+- iovec regs_iovec = {®s, sizeof(regs)};
+- ptraceRet = ptrace(PTRACE_GETREGSET, childPid, (void *)NT_PRSTATUS, ®s_iovec);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Process %u instruction pointer is at %llx, for entry point %p", childPid,
+- (uint64_t)(regs.INST_PTR_REG), entry);
+-
+- // step back past the byte(s) we inserted the breakpoint on
+- regs.INST_PTR_REG -= BREAK_INST_INST_PTR_ADJUST;
+- ptraceRet = ptrace(PTRACE_SETREGSET, childPid, (void *)NT_PRSTATUS, ®s_iovec);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- // restore the function
+- ptraceRet = ptrace(PTRACE_POKETEXT, childPid, entry, origEntryWord);
+- RDCASSERTEQUAL(ptraceRet, 0);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Process %u instruction pointer adjusted and breakpoint removed.", childPid);
+-
+- // we'll resume after reading the ident port in the calling function
+- return true;
++ int stat;
++ pid_t pid;
++ pid = waitpid(childPid, &stat, WUNTRACED);
++ return pid == childPid && WIFSTOPPED(stat);
+ }
+
+-void StopAtMainInChild()
++void ResumeProcess(pid_t childPid, uint32_t delay = 0)
+ {
+- // don't do this unless the ptrace scope is OK.
+- if(!ptrace_scope_ok())
+- return;
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Stopping in main at child for ptracing");
+-
+- // allow parent tracing, and immediately stop so the parent process can attach
+- ptrace(PTRACE_TRACEME, 0, 0, 0);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Done PTRACE_TRACEME, raising SIGSTOP");
+-
+- raise(SIGSTOP);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Resumed after SIGSTOP");
++ kill(childPid, SIGCONT);
+ }
+
+-void ResumeProcess(pid_t childPid, uint32_t delaySeconds)
++void StopAtMainInChild(void)
+ {
+- if(!ptrace_scope_ok())
+- return;
+-
+- if(childPid != 0)
+- {
+- // if we have a delay, see if the process is paused. If so then detach it but keep it stopped
+- // and wait to see if someone attaches
+- if(delaySeconds > 0)
+- {
+- uint64_t ip = get_child_ip(childPid);
+-
+- if(ip != 0)
+- {
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Detaching %u with SIGSTOP to allow a debugger to attach, waiting %u seconds",
+- childPid, delaySeconds);
+-
+- // detach but stop, to allow a debugger to attach
+- ptrace(PTRACE_DETACH, childPid, NULL, SIGSTOP);
+-
+- rdcstr filename = StringFormat::Fmt("/proc/%u/status", childPid);
+-
+- uint64_t start_nano = get_nanotime();
+- uint64_t end_nano = 0;
+-
+- const uint64_t timeoutNanoseconds = uint64_t(delaySeconds) * 1000 * 1000 * 1000;
+-
+- bool connected = false;
+-
+- // watch for a tracer to attach
+- do
+- {
+- usleep(10);
+-
+- rdcstr status;
+- FileIO::ReadAll(filename, status);
+-
+- int32_t offs = status.find("TracerPid:");
+-
+- if(offs < 0)
+- break;
+-
+- status.erase(0, offs + sizeof("TracerPid:"));
+- status.trim();
+-
+- end_nano = get_nanotime();
+-
+- if(status[0] != '0')
+- {
+- RDCLOG("Debugger PID %u attached after %f seconds", atoi(status.c_str()),
+- double(end_nano - start_nano) / 1000000000.0);
+- connected = true;
+- break;
+- }
+- } while(end_nano - start_nano < timeoutNanoseconds);
+-
+- if(!connected)
+- {
+- RDCLOG("Timed out waiting for debugger, resuming");
+- kill(childPid, SIGCONT);
+- }
+- return;
+- }
+- else
+- {
+- RDCERR("Can't delay for debugger without ptrace, check ptrace_scope value");
+- }
+- }
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Detaching immediately from %u", childPid);
+-
+- // try to detach and resume the process, ignoring any errors if we weren't tracing
+- long ret = ptrace(PTRACE_DETACH, childPid, NULL, NULL);
+-
+- if(Linux_Debug_PtraceLogging())
+- RDCLOG("Detached pid %u (%ld)", childPid, ret);
+- }
++ raise(SIGSTOP);
+ }
+
+ // because OSUtility::DebuggerPresent is called often we want it to be
+diff --git a/renderdoc/os/posix/posix_process.cpp b/renderdoc/os/posix/posix_process.cpp
+index 01cc81c8a9..a99d924ec0 100644
+--- a/renderdoc/os/posix/posix_process.cpp
++++ b/renderdoc/os/posix/posix_process.cpp
+@@ -606,9 +606,6 @@ static pid_t RunProcess(rdcstr appName, rdcstr workDir, const rdcstr &cmdLine, c
+ childPid = fork();
+ if(childPid == 0)
+ {
+- if(pauseAtMain)
+- StopAtMainInChild();
+-
+ FileIO::ReleaseFDAfterFork();
+ if(stdoutPipe)
+ {