feat(linux/vaapi): VAAPI encoder improvements (#5388)

docs/configuration.md

@@ -3006,6 +3006,125 @@ editing the `conf` file in a text editor. Use the examples as reference.
 
 ## VA-API Encoder
 
+### vaapi_blbrc
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            Block level based bitrate control (BLBRC) can assign different bitrate on a per-block basis. May improve quality on supported devices.
+            @note{This option only applies when using the VA-API [encoder](#encoder).}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">@code{}
+            disabled
+            @endcode</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            vaapi_blbrc = enabled
+            @endcode</td>
+    </tr>
+</table>
+
+### vaapi_quality
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            The quality profile controls the tradeoff between speed and quality of encoding.
+            @note{This option only applies when using the VA-API [encoder](#encoder).}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">@code{}
+            auto
+            @endcode</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            vaapi_quality = auto
+            @endcode</td>
+    </tr>
+    <tr>
+        <td rowspan="4">Choices</td>
+        <td>auto</td>
+        <td>driver default quality</td>
+    </tr>
+    <tr>
+        <td>speed</td>
+        <td>prefer speed</td>
+    </tr>
+    <tr>
+        <td>balanced</td>
+        <td>balanced</td>
+    </tr>
+    <tr>
+        <td>quality</td>
+        <td>prefer quality</td>
+    </tr>
+</table>
+
+### vaapi_rc
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            The encoder rate control.
+            @note{This option only applies when using the VA-API [encoder](#encoder).}
+            @warning{The automatic setting may override the driver-default rate control method to VBR and force [vaapi_strict_rc_buffer](#vaapi_strict_rc_buffer) enabled on certain configurations. Selecting another rate control manually will override this behaviour.}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">@code{}
+            auto
+            @endcode</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            vaapi_rc = vbr
+            @endcode</td>
+    </tr>
+    <tr>
+        <td rowspan="7">Choices</td>
+        <td>auto</td>
+        <td>driver default (or whitelisted override)</td>
+    </tr>
+        <tr>
+        <td>avbr</td>
+        <td>average variable bitrate</td>
+    </tr>
+    <tr>
+        <td>cbr</td>
+        <td>constant bitrate</td>
+    </tr>
+        <tr>
+        <td>cqp</td>
+        <td>constant qp mode</td>
+    </tr>
+    <tr>
+        <td>icq</td>
+        <td>intelligent qp mode</td>
+    </tr>
+    <tr>
+        <td>qvbr</td>
+        <td>quality-defined variable bitrate</td>
+    </tr>
+        <tr>
+        <td>vbr</td>
+        <td>variable bitrate</td>
+    </tr>
+</table>
+
 ### vaapi_strict_rc_buffer
 
 <table>
@@ -3014,7 +3133,7 @@ editing the `conf` file in a text editor. Use the examples as reference.
         <td colspan="2">
             Enabling this option can avoid dropped frames over the network during scene changes, but video quality may
             be reduced during motion.
-            @note{This option only applies for H.264 and HEVC when using VA-API [encoder](#encoder) on AMD GPUs.}
+            @note{This option only applies for H.264 and HEVC when using VA-API [encoder](#encoder) on AMD GPUs (or when overriding the default rate control on other devices).}
         </td>
     </tr>
     <tr>

src/config.cpp

@@ -38,6 +38,11 @@
   #include <ffnvcodec/nvEncodeAPI.h>
 #endif
 
+#if (defined(linux) || defined(__FreeBSD__)) && !defined(DOXYGEN)
+  // For VAAPI rate control types
+  #include <va/va.h>
+#endif
+
 namespace fs = std::filesystem;
 using namespace std::literals;
 
@@ -397,6 +402,89 @@ namespace config {
 
   }  // namespace qsv
 
+  namespace vaapi {
+#if !(defined(linux) || defined(__FreeBSD__)) || defined(DOXYGEN)
+    constexpr int VA_RC_CBR = 0x00000002;  ///< CBR rate control
+    constexpr int VA_RC_VBR = 0x00000004;  ///< VBR rate control
+    constexpr int VA_RC_CQP = 0x00000010;  ///< CQP rate control
+    constexpr int VA_RC_ICQ = 0x00000040;  ///< ICQ rate control
+    constexpr int VA_RC_QVBR = 0x00000400;  ///< QVBR rate control
+    constexpr int VA_RC_AVBR = 0x00000800;  ///< AVBR rate control
+#endif
+    /**
+     * @brief Enumerates supported VA-API quality options.
+     */
+    enum class quality_e : int {
+      _auto = 0,  ///< Auto quality level
+      speed = 1,  ///< Speed level
+      balanced = 2,  ///< Balanced level
+      quality = 3  ///< Quality level
+    };
+
+    /**
+     * @brief Enumerates supported VA-API rc options.
+     */
+    enum class rc_e : int {
+      _auto = 0,  ///< Auto rate control
+      avbr = VA_RC_AVBR,  ///< AVBR - average variable bitrate
+      cbr = VA_RC_CBR,  ///< CBR - constant bitrate
+      cqp = VA_RC_CQP,  ///< CQP - constant QP
+      icq = VA_RC_ICQ,  ///< ICQ - intelligent QP
+      qvbr = VA_RC_QVBR,  ///< QVBR - quality-defined variable bitrate
+      vbr = VA_RC_VBR  ///< VBR - variable bitrate
+    };
+
+    /**
+     * @brief Parse a VA-API quality preset while preserving the current value on invalid input.
+     *
+     * @param quality_type Configuration text naming the VA-API quality preset.
+     * @param original Original text value used when reporting a parsing failure.
+     * @return Parsed enum value, or the setting-specific default when the text is unknown.
+     */
+    template<class T>
+    ::std::optional<int> quality_from_view(const ::std::string_view &quality_type, const ::std::optional<int>(&original)) {
+#ifndef DOXYGEN
+  #define _CONVERT_(x) \
+    if (quality_type == #x##sv) \
+    return (int) T::x
+#endif
+      _CONVERT_(balanced);
+      _CONVERT_(quality);
+      _CONVERT_(speed);
+#ifdef _CONVERT_
+  #undef _CONVERT_
+#endif
+      return original;
+    }
+
+    /**
+     * @brief Parse a VA-API rate-control mode while preserving the current value on invalid input.
+     *
+     * @param rc Rate-control mode selected in the configuration.
+     * @param original Original text value used when reporting a parsing failure.
+     * @return Parsed enum value, or the setting-specific default when the text is unknown.
+     */
+    template<class T>
+    ::std::optional<int> rc_from_view(const ::std::string_view &rc, const ::std::optional<int>(&original)) {
+#ifndef DOXYGEN
+  #define _CONVERT_(x) \
+    if (rc == #x##sv) \
+    return (int) T::x
+#endif
+      _CONVERT_(avbr);
+      _CONVERT_(cbr);
+      _CONVERT_(cqp);
+      _CONVERT_(icq);
+      _CONVERT_(qvbr);
+      _CONVERT_(vbr);
+#ifdef _CONVERT_
+  #undef _CONVERT_
+#endif
+      return original;
+    }
+
+  }  // namespace vaapi
+
   namespace vt {
 
     /**
@@ -669,6 +757,10 @@ namespace config {
     },  // vt
 
     {
+      0,  // blbrc
+      std::to_underlying(vaapi::quality_e::_auto),  // quality
+      std::to_underlying(vaapi::rc_e::_auto),  // rate control
+      {},  // rate control string
       false,  // strict_rc_buffer
     },  // vaapi
 
@@ -1551,6 +1643,16 @@ namespace config {
     int_f(vars, "vt_software", video.vt.vt_require_sw, vt::force_software_from_view);
     int_f(vars, "vt_realtime", video.vt.vt_realtime, vt::rt_from_view);
 
+    std::string vaapi_quality;
+    string_f(vars, "vaapi_quality", vaapi_quality);
+    if (!vaapi_quality.empty()) {
+      video.vaapi.vaapi_quality = vaapi::quality_from_view<vaapi::quality_e>(vaapi_quality, video.vaapi.vaapi_quality);
+    }
+    string_f(vars, "vaapi_rc", video.vaapi.vaapi_rc_str);
+    if (!video.vaapi.vaapi_rc_str.empty()) {
+      video.vaapi.vaapi_rc = vaapi::rc_from_view<vaapi::rc_e>(video.vaapi.vaapi_rc_str, video.vaapi.vaapi_rc);
+    }
+    bool_f(vars, "vaapi_blbrc", (bool &) video.vaapi.blbrc);
     bool_f(vars, "vaapi_strict_rc_buffer", video.vaapi.strict_rc_buffer);
 
     int_f(vars, "vk_tune", video.vk.tune);

src/config.h

@@ -105,6 +105,10 @@ namespace config {
     } vt;  ///< VideoToolbox encoder options.
 
     struct {
+      std::optional<int> blbrc;
+      std::optional<int> vaapi_quality;
+      std::optional<int> vaapi_rc;
+      std::string vaapi_rc_str;
       bool strict_rc_buffer;
     } vaapi;  ///< VA-API encoder options.
 

src/platform/linux/vaapi.cpp

@@ -308,8 +308,33 @@ namespace va {
         BOOST_LOG(info) << "Using normal encoding mode"sv;
       }
 
+      // When the compression_level AVOption is set, vaapi_encode.c assigns the value to VAEncMiscParameterBufferQualityLevel
+      VAConfigAttrib quality_attr = {VAConfigAttribEncQualityRange};
+      auto status = vaGetConfigAttributes(va_display, va_profile, va_entrypoint, &quality_attr, 1);
+      if (status != VA_STATUS_SUCCESS || quality_attr.value == VA_ATTRIB_NOT_SUPPORTED) {
+        quality_attr.value = 0;
+      }
+      auto vaapi_quality = config::video.vaapi.vaapi_quality.value_or(0);
+      auto target_quality = 0;
+      switch (vaapi_quality) {
+        default:
+        case 0:  // auto or unset
+          break;
+        case 1:  // low quality (highest value in range)
+        case 2:  // med quality (middle value in range)
+          target_quality = quality_attr.value / vaapi_quality;
+          break;
+        case 3:  // high quality (1)
+          target_quality = 1;
+          break;
+      }
+      if (quality_attr.value > 0) {
+        ctx->compression_level = target_quality;
+        BOOST_LOG(info) << "[VAAPI] Quality level set to "sv << ctx->compression_level << " (fastest level: "sv << quality_attr.value << ")"sv;
+      }
+
       VAConfigAttrib rc_attr = {VAConfigAttribRateControl};
-      auto status = vaGetConfigAttributes(va_display, va_profile, va_entrypoint, &rc_attr, 1);
+      status = vaGetConfigAttributes(va_display, va_profile, va_entrypoint, &rc_attr, 1);
       if (status != VA_STATUS_SUCCESS) {
         // Stick to the default rate control (CQP)
         rc_attr.value = 0;
@@ -337,27 +362,52 @@ namespace va {
       // When we have to resort to the default 1 second VBV for encoding quality reasons,
       // we stick to CBR in order to avoid encoding huge frames after bitrate undershoots
       // leave headroom available in the RC window.
-      if (config::video.vaapi.strict_rc_buffer ||
+      //
-          (vendor && strstr(vendor, "Intel")) ||
+      // If a user-supplied rate control is detected, override the whitelist logic to allow
-          ctx->codec_id == AV_CODEC_ID_AV1) {
+      // full user control of both the rate control and strict VBV settings.
-        ctx->rc_buffer_size = ctx->bit_rate * ctx->framerate.den / ctx->framerate.num;
+      auto auto_whitelist = false;
+      auto rc_mode = config::video.vaapi.vaapi_rc_str;
+      auto rc_vbv = "with standard VBV size";
+      auto rc_whitelist = "";
+      auto rc_val = config::video.vaapi.vaapi_rc.value_or(0);
 
-        if (rc_attr.value & VA_RC_VBR) {
+      // Detect whitelisted configurations
-          BOOST_LOG(info) << "Using VBR with single frame VBV size"sv;
+      if ((vendor && std::string_view(vendor).contains("Intel") == true) || ctx->codec_id == AV_CODEC_ID_AV1) {
-          av_dict_set(options, "rc_mode", "VBR", 0);
+        auto_whitelist = true;
-        } else if (rc_attr.value & VA_RC_CBR) {
+        rc_whitelist = " (whitelist override)";
-          BOOST_LOG(info) << "Using CBR with single frame VBV size"sv;
-          av_dict_set(options, "rc_mode", "CBR", 0);
-        } else {
-          BOOST_LOG(warning) << "Using CQP with single frame VBV size"sv;
-          av_dict_set_int(options, "qp", config::video.qp, 0);
-        }
-      } else if (!(rc_attr.value & (VA_RC_CBR | VA_RC_VBR))) {
-        BOOST_LOG(warning) << "Using CQP rate control"sv;
-        av_dict_set_int(options, "qp", config::video.qp, 0);
-      } else {
-        BOOST_LOG(info) << "Using default rate control"sv;
       }
+
+      // First try user config, else fall back to auto-detection that respects whitelist
+      if (rc_val > 0 && rc_attr.value & rc_val) {
+        // override whitelist if the user-specified RC is supported
+        auto_whitelist = false;
+        rc_whitelist = "";
+      } else if (rc_attr.value & VA_RC_VBR && auto_whitelist) {
+        rc_mode = "vbr";
+        rc_val = VA_RC_VBR;
+      } else if (rc_attr.value & VA_RC_CBR) {
+        rc_mode = "cbr";
+        rc_val = VA_RC_CBR;
+      } else {
+        rc_mode = "cqp";
+        rc_val = VA_RC_CQP;
+      }
+
+      if (config::video.vaapi.strict_rc_buffer || auto_whitelist) {
+        ctx->rc_buffer_size = ctx->bit_rate * ctx->framerate.den / ctx->framerate.num;
+        rc_vbv = "with single frame VBV size";
+      }
+
+      // ffmpeg's rc_mode values don't align with VAAPI's rc_val values, so transform string to uppercase
+      std::transform(rc_mode.begin(), rc_mode.end(), rc_mode.begin(), [](unsigned char c) {
+        return std::toupper(c);
+      });
+      av_dict_set(options, "rc_mode", rc_mode.c_str(), 0);
+      if (rc_val == VA_RC_CQP || rc_val == VA_RC_ICQ || rc_val == VA_RC_QVBR) {
+        BOOST_LOG(warning) << "[VAAPI] Applying QP for compatible rate control method (QP value: "sv << config::video.qp << ")"sv;
+        av_dict_set_int(options, "qp", config::video.qp, 0);
+      }
+      BOOST_LOG(info) << "[VAAPI] Using "sv << rc_mode << " rate control "sv << rc_vbv << rc_whitelist;
     }
 
     /**

src/video.cpp

@@ -1191,6 +1191,7 @@ namespace video {
       // Common options
       {
         {"async_depth"s, 1},
+        {"blbrc"s, &config::video.vaapi.blbrc},
         {"idr_interval"s, std::numeric_limits<int>::max()},
       },
       {},  // SDR-specific options
@@ -1204,6 +1205,7 @@ namespace video {
       // Common options
       {
         {"async_depth"s, 1},
+        {"blbrc"s, &config::video.vaapi.blbrc},
         {"sei"s, 0},
         {"idr_interval"s, std::numeric_limits<int>::max()},
       },
@@ -1218,6 +1220,7 @@ namespace video {
       // Common options
       {
         {"async_depth"s, 1},
+        {"blbrc"s, &config::video.vaapi.blbrc},
         {"sei"s, 0},
         {"idr_interval"s, std::numeric_limits<int>::max()},
       },

src_assets/common/assets/web/config.html

@@ -356,6 +356,9 @@
             id: "vaapi",
             name: "VA-API Encoder",
             options: {
+              "vaapi_blbrc": "disabled",
+              "vaapi_quality": "auto",
+              "vaapi_rc": "auto",
               "vaapi_strict_rc_buffer": "disabled",
             },
           },

src_assets/common/assets/web/configs/tabs/encoders/VAAPIEncoder.vue

@@ -12,13 +12,80 @@ const config = ref(props.config)
 
 <template>
   <div id="vaapi-encoder" class="config-page">
-    <!-- Strict RC Buffer -->
+    <!-- VAAPI Rate Control group options -->
-    <Checkbox class="mb-3"
+    <div class="mb-3 accordion">
-              id="vaapi_strict_rc_buffer"
+      <div class="accordion-item">
-              locale-prefix="config"
+        <h2 class="accordion-header">
-              v-model="config.vaapi_strict_rc_buffer"
+          <button class="accordion-button" type="button" data-bs-toggle="collapse"
-              default="false"
+                  data-bs-target="#panelsStayOpen-collapseOne">
-    ></Checkbox>
+            {{ $t('config.vaapi_rc_group') }}
+          </button>
+        </h2>
+        <div id="panelsStayOpen-collapseOne" class="accordion-collapse collapse show"
+             aria-labelledby="panelsStayOpen-headingOne">
+          <div class="accordion-body">
+            <!-- VAAPI Rate Control -->
+            <div class="mb-3">
+              <label for="vaapi_rc" class="form-label">{{ $t('config.vaapi_rc') }}</label>
+              <select id="vaapi_rc" class="form-select" v-model="config.vaapi_rc">
+                <option value="auto">{{ $t('auto') }}</option>
+                <option value="avbr">{{ $t('config.vaapi_rc_avbr') }}</option>
+                <option value="vbr">{{ $t('config.vaapi_rc_vbr') }}</option>
+                <option value="cbr">{{ $t('config.vaapi_rc_cbr') }}</option>
+                <option value="cqp">{{ $t('config.vaapi_rc_cqp') }}</option>
+                <option value="icq">{{ $t('config.vaapi_rc_icq') }}</option>
+                <option value="qvbr">{{ $t('config.vaapi_rc_qvbr') }}</option>
+              </select>
+              <div class="form-text">{{ $t('config.vaapi_rc_desc') }}</div>
+            </div>
+
+            <!-- BLBRC -->
+            <Checkbox class="mb-3"
+                      id="vaapi_blbrc"
+                      locale-prefix="config"
+                      v-model="config.vaapi_blbrc"
+                      default="false"
+            ></Checkbox>
+
+            <!-- Strict RC Buffer -->
+            <Checkbox class="mb-3"
+                      id="vaapi_strict_rc_buffer"
+                      locale-prefix="config"
+                      v-model="config.vaapi_strict_rc_buffer"
+                      default="false"
+            ></Checkbox>
+          </div>
+        </div>
+      </div>
+    </div>
+
+    <!-- VAAPI Quality group options -->
+    <div class="mb-3 accordion">
+      <div class="accordion-item">
+        <h2 class="accordion-header">
+          <button class="accordion-button" type="button" data-bs-toggle="collapse"
+                  data-bs-target="#panelsStayOpen-collapseTwo">
+            {{ $t('config.vaapi_quality_group') }}
+          </button>
+        </h2>
+        <div id="panelsStayOpen-collapseTwo" class="accordion-collapse collapse show"
+             aria-labelledby="panelsStayOpen-headingTwo">
+          <div class="accordion-body">
+            <!-- VAAPI Quality -->
+            <div class="mb-3">
+              <label for="vaapi_quality" class="form-label">{{ $t('config.vaapi_quality') }}</label>
+              <select id="vaapi_quality" class="form-select" v-model="config.vaapi_quality">
+                <option value="auto">{{ $t('auto') }}</option>
+                <option value="speed">{{ $t('config.vaapi_quality_speed') }}</option>
+                <option value="balanced">{{ $t('config.vaapi_quality_balanced') }}</option>
+                <option value="quality">{{ $t('config.vaapi_quality_quality') }}</option>
+              </select>
+              <div class="form-text">{{ $t('config.vaapi_quality_desc') }}</div>
+            </div>
+          </div>
+        </div>
+      </div>
+    </div>
   </div>
 </template>
 

src_assets/common/assets/web/public/assets/locale/en.json

@@ -397,6 +397,23 @@
     "touchpad_as_ds4_desc": "If disabled, touchpad presence will not be taken into account during gamepad type selection.",
     "upnp": "UPnP",
     "upnp_desc": "Automatically configure port forwarding for streaming over the Internet",
+    "vaapi_blbrc": "Block level based bitrate control (BLBRC)",
+    "vaapi_blbrc_desc": "Enable block level rate control, which assigns different bitrate block by block. Can be combined with CBR/VBR-type rate control methods on supported encoders.",
+    "vaapi_quality": "VA-API Quality",
+    "vaapi_quality_balanced": "balanced -- balanced",
+    "vaapi_quality_desc": "Determines encoder tradeoff between quality and speed/power consumption.",
+    "vaapi_quality_group": "VA-API Quality Settings",
+    "vaapi_quality_quality": "quality -- prefer quality",
+    "vaapi_quality_speed": "speed -- prefer speed",
+    "vaapi_rc": "VA-API Rate Control",
+    "vaapi_rc_avbr": "avbr -- average variable bitrate",
+    "vaapi_rc_cbr": "cbr -- constant bitrate",
+    "vaapi_rc_cqp": "cqp -- constant qp mode",
+    "vaapi_rc_desc": "This controls the rate control method to ensure we are not exceeding the client bitrate target. Note that supported rate control methods varies by vendor, and automatic may force-enable strict bitrate enforcement on certain devices.",
+    "vaapi_rc_group": "VA-API Rate Control Settings",
+    "vaapi_rc_icq": "icq -- intelligent constant qp mode",
+    "vaapi_rc_qvbr": "qvbr -- quality-defined variable bitrate",
+    "vaapi_rc_vbr": "vbr -- variable bitrate",
     "vaapi_strict_rc_buffer": "Strictly enforce frame bitrate limits for H.264/HEVC on AMD GPUs",
     "vaapi_strict_rc_buffer_desc": "Enabling this option can avoid dropped frames over the network during scene changes, but video quality may be reduced during motion.",
     "vk_rc_cbr": "CBR (Constant Bitrate) (default)",