fix(linux): drop implicit DRM master for card fds (#5286)

Co-authored-by: Kishi85 <41839133+Kishi85@users.noreply.github.com>

src/platform/linux/cuda.cpp

@@ -383,7 +383,7 @@ namespace cuda {
 
         fs::path dri_path {"/dev/dri"sv};
         auto device_path = dri_path / file;
-        return open(device_path.c_str(), O_RDWR);
+        return platf::open_drm_card_fd(device_path);
       }
     } catch (const std::filesystem::filesystem_error &err) {
       BOOST_LOG(error) << "Failed to read sysfs: "sv << err.what();

src/platform/linux/kmsgrab.cpp

@@ -436,10 +436,8 @@ namespace platf {
        */
       int init(const char *path) {
         cap_sys_admin admin;
-        fd.el = open(path, O_RDWR);
-
+        fd.el = open_drm_card_fd(path);
         if (fd.el < 0) {
-          BOOST_LOG(error) << "Couldn't open: "sv << path << ": "sv << strerror(errno);
           return -1;
         }
 

src/platform/linux/misc.cpp

@@ -13,6 +13,8 @@
 #endif
 
 // standard includes
+#include <cerrno>
+#include <cstring>
 #include <fstream>
 #include <iomanip>
 #include <iostream>
@@ -154,6 +156,80 @@ namespace platf {
    */
   using ifaddr_t = util::safe_ptr<ifaddrs, freeifaddrs>;
 
+  /**
+   * @brief Open a DRM card node, dropping implicit DRM master when possible.
+   *
+   * See `misc.h` for full documentation. Master check/drop failures are logged
+   * as warnings but do not fail the call.
+   */
+  int open_drm_card_fd(const std::filesystem::path &path, int flags) {
+#ifdef SUNSHINE_BUILD_DRM
+    int fd = open(path.c_str(), flags | O_CLOEXEC);
+    if (fd < 0) {
+      BOOST_LOG(error) << "Couldn't open: "sv << path.string() << ": "sv << strerror(errno);
+      return -1;
+    }
+
+    auto is_master = [&]() -> int {
+      drm_auth_t auth {};
+      auth.magic = 0;
+
+      errno = 0;
+      if (drmIoctl(fd, DRM_IOCTL_AUTH_MAGIC, &auth) == 0) {
+        return 1;  ///< AUTH_MAGIC succeeded, so we are master.
+      }
+
+      auto err = errno;
+      if (err == EACCES) {
+        return 0;  ///< Kernel rejected the ioctl because we are not master.
+      }
+
+      if (err == EINVAL || err == ENOENT) {
+        return 1;  ///< Ioctl reached the master path but the magic (0) was invalid; we are master.
+      }
+
+      BOOST_LOG(warning) << "Couldn't determine DRM master state for "sv << path.string() << ": "sv << strerror(err);
+      return -1;
+    };
+
+    auto master = is_master();
+    if (master < 0) {
+      BOOST_LOG(warning) << "Proceeding without dropping DRM master for "sv << path.string()
+                         << "; compositor VT switches may fail."sv;
+      return fd;
+    }
+
+    if (master) {
+      if (drmDropMaster(fd)) {
+        auto err = errno;
+        BOOST_LOG(warning) << "Couldn't drop DRM master for "sv << path.string() << ": "sv << strerror(err)
+                           << ", Compositor VT switches may fail."sv;
+        return fd;
+      }
+
+      BOOST_LOG(info) << "Dropped DRM master for "sv << path.string();
+
+      master = is_master();
+      if (master < 0) {
+        BOOST_LOG(warning) << "Could not re-verify DRM master state after drop for "sv << path.string() << "."sv;
+        return fd;
+      }
+
+      if (master) {
+        BOOST_LOG(warning) << "Still DRM master after drop for "sv << path.string() << "."sv;
+        return fd;
+      }
+    }
+
+    return fd;
+#else
+  #ifndef __FreeBSD__
+    BOOST_LOG(info) << "Sunshine compiled without DRM support. Cannot control Linux DRM master state for "sv << path.string();
+  #endif
+    return open(path.c_str(), flags | O_CLOEXEC);
+#endif
+  }
+
   /**
    * @brief Read the local interface address list.
    *

src/platform/linux/misc.h

@@ -5,6 +5,8 @@
 #pragma once
 
 // standard includes
+#include <fcntl.h>
+#include <filesystem>
 #include <unistd.h>
 #include <vector>
 
@@ -45,3 +47,24 @@ namespace dyn {
   void *handle(const std::vector<const char *> &libs);
 
 }  // namespace dyn
+
+namespace platf {
+  /**
+   * @brief Open a DRM card node and drop implicit DRM master, if any.
+   *
+   * Performs `open(path, flags | O_CLOEXEC)` and probes the resulting fd with
+   * `DRM_IOCTL_AUTH_MAGIC`. If the kernel implicitly handed us master, calls
+   * `drmDropMaster` and re-verifies before returning. Master check/drop failures
+   * are logged as warnings but do not fail the call: the caller still receives
+   * a usable fd.
+   *
+   * Callers should use this helper for any `/dev/dri/cardN` open so we never
+   * keep implicit master and block compositors from re-acquiring it on VT
+   * switches.
+   *
+   * @param path Path to the DRM card node (e.g. `/dev/dri/card0`).
+   * @param flags `open()` flags. `O_CLOEXEC` is always OR-ed in.
+   * @return A file descriptor on success, or `-1` if `open()` itself fails.
+   */
+  int open_drm_card_fd(const std::filesystem::path &path, int flags = O_RDWR);
+}  // namespace platf