From 3e6c835a64e99e36f09e39a2a5a34b19f244d3d4 Mon Sep 17 00:00:00 2001
From: lat9nq <22451773+lat9nq@users.noreply.github.com>
Date: Wed, 23 Nov 2022 21:35:47 -0500
Subject: [PATCH 1/2] startup_checks: Use Windows flow for *nix

Spawns a child using fork and exec as opposed to fork alone. Workaround
for the macos file manager complaining about not supporting fork without
exec.

Control flow for *nix is now roughly the same as for Windows.
---
 src/yuzu/startup_checks.cpp | 56 +++++++++++++++++++++++++++++++------
 src/yuzu/startup_checks.h   |  4 +++
 2 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
index 6a91212e2..95c9fdecb 100644
--- a/src/yuzu/startup_checks.cpp
+++ b/src/yuzu/startup_checks.cpp
@@ -4,11 +4,14 @@
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
 #ifdef _WIN32
-#include <cstring> // for memset, strncpy
+#include <cstring> // for memset, strncpy, strncmp
 #include <processthreadsapi.h>
 #include <windows.h>
 #elif defined(YUZU_UNIX)
+#include <cstring> // for strncmp
 #include <errno.h>
+#include <spawn.h>
+#include <sys/types.h>
 #include <sys/wait.h>
 #include <unistd.h>
 #endif
@@ -53,6 +56,13 @@ bool CheckEnvVars(bool* is_child) {
                      IS_CHILD_ENV_VAR, GetLastError());
         return true;
     }
+#elif defined(YUZU_UNIX)
+    const char* startup_check_var = getenv(STARTUP_CHECK_ENV_VAR);
+    if (startup_check_var != nullptr &&
+        std::strncmp(startup_check_var, ENV_VAR_ENABLED_TEXT, 8) == 0) {
+        CheckVulkan();
+        return true;
+    }
 #endif
     return false;
 }
@@ -101,20 +111,22 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
     }
 
 #elif defined(YUZU_UNIX)
+    const int env_var_set = setenv(STARTUP_CHECK_ENV_VAR, ENV_VAR_ENABLED_TEXT, 1);
+    if (env_var_set == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "setenv failed to set %s with error %d\n", STARTUP_CHECK_ENV_VAR, err);
+        return false;
+    }
+
     if (perform_vulkan_check) {
-        const pid_t pid = fork();
-        if (pid == 0) {
-            CheckVulkan();
-            return true;
-        } else if (pid == -1) {
-            const int err = errno;
-            std::fprintf(stderr, "fork failed with error %d\n", err);
+        const pid_t pid = SpawnChild(arg0);
+        if (pid == -1) {
             return false;
         }
 
         // Get exit code from child process
         int status;
-        const int r_val = wait(&status);
+        const int r_val = waitpid(pid, &status, 0);
         if (r_val == -1) {
             const int err = errno;
             std::fprintf(stderr, "wait failed with error %d\n", err);
@@ -123,6 +135,13 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
         // Vulkan is broken if the child crashed (return value is not zero)
         *has_broken_vulkan = (status != 0);
     }
+
+    const int env_var_cleared = unsetenv(STARTUP_CHECK_ENV_VAR);
+    if (env_var_cleared == -1) {
+        const int err = errno;
+        std::fprintf(stderr, "unsetenv failed to clear %s with error %d\n", STARTUP_CHECK_ENV_VAR,
+                     err);
+    }
 #endif
     return false;
 }
@@ -156,4 +175,23 @@ bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags) {
 
     return true;
 }
+#elif defined(YUZU_UNIX)
+pid_t SpawnChild(const char* arg0) {
+    const pid_t pid = fork();
+
+    if (pid == -1) {
+        // error
+        const int err = errno;
+        std::fprintf(stderr, "fork failed with error %d\n", err);
+        return pid;
+    } else if (pid == 0) {
+        // child
+        execl(arg0, arg0, nullptr);
+        const int err = errno;
+        std::fprintf(stderr, "execl failed with error %d\n", err);
+        return -1;
+    }
+
+    return pid;
+}
 #endif
diff --git a/src/yuzu/startup_checks.h b/src/yuzu/startup_checks.h
index d8e563be6..2f86fb843 100644
--- a/src/yuzu/startup_checks.h
+++ b/src/yuzu/startup_checks.h
@@ -5,6 +5,8 @@
 
 #ifdef _WIN32
 #include <windows.h>
+#elif defined(YUZU_UNIX)
+#include <sys/types.h>
 #endif
 
 constexpr char IS_CHILD_ENV_VAR[] = "YUZU_IS_CHILD";
@@ -17,4 +19,6 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
 
 #ifdef _WIN32
 bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags);
+#elif defined(YUZU_UNIX)
+pid_t SpawnChild(const char* arg0);
 #endif

From 35319ca3a5d6f6dd2b7619ca38d6f2b4696ff3dd Mon Sep 17 00:00:00 2001
From: lat9nq <22451773+lat9nq@users.noreply.github.com>
Date: Wed, 23 Nov 2022 21:57:28 -0500
Subject: [PATCH 2/2] startup_checks: Use fmt::print, fix exec error handling

Uses fmt::print opposed to std::fprintf for error printing.

Call exit instead of returning to caller to prevent a like issue the
previous commit was trying to solve.

Removes unneeded comment.

Co-authored-by: liamwhite <liamwhite@users.noreply.github.com>
Co-authored-by: Lioncash <mathew1800@gmail.com>
---
 src/yuzu/startup_checks.cpp | 42 ++++++++++++++++++-------------------
 1 file changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/yuzu/startup_checks.cpp b/src/yuzu/startup_checks.cpp
index 95c9fdecb..ccdcf10fa 100644
--- a/src/yuzu/startup_checks.cpp
+++ b/src/yuzu/startup_checks.cpp
@@ -4,11 +4,11 @@
 #include "video_core/vulkan_common/vulkan_wrapper.h"
 
 #ifdef _WIN32
-#include <cstring> // for memset, strncpy, strncmp
+#include <cstring>
 #include <processthreadsapi.h>
 #include <windows.h>
 #elif defined(YUZU_UNIX)
-#include <cstring> // for strncmp
+#include <cstring>
 #include <errno.h>
 #include <spawn.h>
 #include <sys/types.h>
@@ -16,7 +16,7 @@
 #include <unistd.h>
 #endif
 
-#include <cstdio>
+#include <fmt/core.h>
 #include "video_core/vulkan_common/vulkan_instance.h"
 #include "video_core/vulkan_common/vulkan_library.h"
 #include "yuzu/startup_checks.h"
@@ -30,7 +30,7 @@ void CheckVulkan() {
             Vulkan::CreateInstance(library, dld, VK_API_VERSION_1_0);
 
     } catch (const Vulkan::vk::Exception& exception) {
-        std::fprintf(stderr, "Failed to initialize Vulkan: %s\n", exception.what());
+        fmt::print(stderr, "Failed to initialize Vulkan: {}\n", exception.what());
     }
 }
 
@@ -52,8 +52,8 @@ bool CheckEnvVars(bool* is_child) {
         *is_child = true;
         return false;
     } else if (!SetEnvironmentVariableA(IS_CHILD_ENV_VAR, ENV_VAR_ENABLED_TEXT)) {
-        std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %lu\n",
-                     IS_CHILD_ENV_VAR, GetLastError());
+        fmt::print(stderr, "SetEnvironmentVariableA failed to set {} with error {}\n",
+                   IS_CHILD_ENV_VAR, GetLastError());
         return true;
     }
 #elif defined(YUZU_UNIX)
@@ -72,8 +72,8 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
     // Set the startup variable for child processes
     const bool env_var_set = SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, ENV_VAR_ENABLED_TEXT);
     if (!env_var_set) {
-        std::fprintf(stderr, "SetEnvironmentVariableA failed to set %s with error %lu\n",
-                     STARTUP_CHECK_ENV_VAR, GetLastError());
+        fmt::print(stderr, "SetEnvironmentVariableA failed to set {} with error {}\n",
+                   STARTUP_CHECK_ENV_VAR, GetLastError());
         return false;
     }
 
@@ -91,30 +91,30 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
         DWORD exit_code = STILL_ACTIVE;
         const int err = GetExitCodeProcess(process_info.hProcess, &exit_code);
         if (err == 0) {
-            std::fprintf(stderr, "GetExitCodeProcess failed with error %lu\n", GetLastError());
+            fmt::print(stderr, "GetExitCodeProcess failed with error {}\n", GetLastError());
         }
 
         // Vulkan is broken if the child crashed (return value is not zero)
         *has_broken_vulkan = (exit_code != 0);
 
         if (CloseHandle(process_info.hProcess) == 0) {
-            std::fprintf(stderr, "CloseHandle failed with error %lu\n", GetLastError());
+            fmt::print(stderr, "CloseHandle failed with error {}\n", GetLastError());
         }
         if (CloseHandle(process_info.hThread) == 0) {
-            std::fprintf(stderr, "CloseHandle failed with error %lu\n", GetLastError());
+            fmt::print(stderr, "CloseHandle failed with error {}\n", GetLastError());
         }
     }
 
     if (!SetEnvironmentVariableA(STARTUP_CHECK_ENV_VAR, nullptr)) {
-        std::fprintf(stderr, "SetEnvironmentVariableA failed to clear %s with error %lu\n",
-                     STARTUP_CHECK_ENV_VAR, GetLastError());
+        fmt::print(stderr, "SetEnvironmentVariableA failed to clear {} with error {}\n",
+                   STARTUP_CHECK_ENV_VAR, GetLastError());
     }
 
 #elif defined(YUZU_UNIX)
     const int env_var_set = setenv(STARTUP_CHECK_ENV_VAR, ENV_VAR_ENABLED_TEXT, 1);
     if (env_var_set == -1) {
         const int err = errno;
-        std::fprintf(stderr, "setenv failed to set %s with error %d\n", STARTUP_CHECK_ENV_VAR, err);
+        fmt::print(stderr, "setenv failed to set {} with error {}\n", STARTUP_CHECK_ENV_VAR, err);
         return false;
     }
 
@@ -129,7 +129,7 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
         const int r_val = waitpid(pid, &status, 0);
         if (r_val == -1) {
             const int err = errno;
-            std::fprintf(stderr, "wait failed with error %d\n", err);
+            fmt::print(stderr, "wait failed with error {}\n", err);
             return false;
         }
         // Vulkan is broken if the child crashed (return value is not zero)
@@ -139,8 +139,8 @@ bool StartupChecks(const char* arg0, bool* has_broken_vulkan, bool perform_vulka
     const int env_var_cleared = unsetenv(STARTUP_CHECK_ENV_VAR);
     if (env_var_cleared == -1) {
         const int err = errno;
-        std::fprintf(stderr, "unsetenv failed to clear %s with error %d\n", STARTUP_CHECK_ENV_VAR,
-                     err);
+        fmt::print(stderr, "unsetenv failed to clear {} with error {}\n", STARTUP_CHECK_ENV_VAR,
+                   err);
     }
 #endif
     return false;
@@ -169,7 +169,7 @@ bool SpawnChild(const char* arg0, PROCESS_INFORMATION* pi, int flags) {
                                                 pi             // lpProcessInformation
     );
     if (!process_created) {
-        std::fprintf(stderr, "CreateProcessA failed with error %lu\n", GetLastError());
+        fmt::print(stderr, "CreateProcessA failed with error {}\n", GetLastError());
         return false;
     }
 
@@ -182,14 +182,14 @@ pid_t SpawnChild(const char* arg0) {
     if (pid == -1) {
         // error
         const int err = errno;
-        std::fprintf(stderr, "fork failed with error %d\n", err);
+        fmt::print(stderr, "fork failed with error {}\n", err);
         return pid;
     } else if (pid == 0) {
         // child
         execl(arg0, arg0, nullptr);
         const int err = errno;
-        std::fprintf(stderr, "execl failed with error %d\n", err);
-        return -1;
+        fmt::print(stderr, "execl failed with error {}\n", err);
+        _exit(0);
     }
 
     return pid;