feat(linux/vaapi): VAAPI encoder improvements

docs/configuration.md

@@ -3006,6 +3006,125 @@ editing the `conf` file in a text editor. Use the examples as reference.
 
 ## VA-API Encoder
 
+### vaapi_blbrc
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            Block level based bitrate control (BLBRC) can assign different bitrate on a per-block basis. May improve quality on supported devices.
+            @note{This option only applies when using the VA-API [encoder](#encoder).}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">@code{}
+            disabled
+            @endcode</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            vaapi_blbrc = enabled
+            @endcode</td>
+    </tr>
+</table>
+
+### vaapi_quality
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            The quality profile controls the tradeoff between speed and quality of encoding.
+            @note{This option only applies when using the VA-API [encoder](#encoder).}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">@code{}
+            auto
+            @endcode</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            vaapi_quality = auto
+            @endcode</td>
+    </tr>
+    <tr>
+        <td rowspan="4">Choices</td>
+        <td>auto</td>
+        <td>driver default quality</td>
+    </tr>
+    <tr>
+        <td>speed</td>
+        <td>prefer speed</td>
+    </tr>
+    <tr>
+        <td>balanced</td>
+        <td>balanced</td>
+    </tr>
+    <tr>
+        <td>quality</td>
+        <td>prefer quality</td>
+    </tr>
+</table>
+
+### vaapi_rc
+
+<table>
+    <tr>
+        <td>Description</td>
+        <td colspan="2">
+            The encoder rate control.
+            @note{This option only applies when using the VA-API [encoder](#encoder).}
+            @warning{The automatic setting may override the driver-default rate control method to VBR and force [vaapi_strict_rc_buffer](#vaapi_strict_rc_buffer) enabled on certain configurations. Selecting another rate control manually will override this behaviour.}
+        </td>
+    </tr>
+    <tr>
+        <td>Default</td>
+        <td colspan="2">@code{}
+            auto
+            @endcode</td>
+    </tr>
+    <tr>
+        <td>Example</td>
+        <td colspan="2">@code{}
+            vaapi_rc = vbr
+            @endcode</td>
+    </tr>
+    <tr>
+        <td rowspan="7">Choices</td>
+        <td>auto</td>
+        <td>driver default (or whitelisted override)</td>
+    </tr>
+        <tr>
+        <td>avbr</td>
+        <td>average variable bitrate</td>
+    </tr>
+    <tr>
+        <td>cbr</td>
+        <td>constant bitrate</td>
+    </tr>
+        <tr>
+        <td>cqp</td>
+        <td>constant qp mode</td>
+    </tr>
+    <tr>
+        <td>icq</td>
+        <td>intelligent qp mode</td>
+    </tr>
+    <tr>
+        <td>qvbr</td>
+        <td>quality-defined variable bitrate</td>
+    </tr>
+        <tr>
+        <td>vbr</td>
+        <td>variable bitrate</td>
+    </tr>
+</table>
+
 ### vaapi_strict_rc_buffer
 
 <table>
@@ -3014,7 +3133,7 @@ editing the `conf` file in a text editor. Use the examples as reference.
         <td colspan="2">
             Enabling this option can avoid dropped frames over the network during scene changes, but video quality may
             be reduced during motion.
-            @note{This option only applies for H.264 and HEVC when using VA-API [encoder](#encoder) on AMD GPUs.}
+            @note{This option only applies for H.264 and HEVC when using VA-API [encoder](#encoder) on AMD GPUs (or when overriding the default rate control on other devices).}
         </td>
     </tr>
     <tr>

src/config.cpp

@@ -38,6 +38,11 @@
   #include <ffnvcodec/nvEncodeAPI.h>
 #endif
 
+#if (defined(linux) || defined(__FreeBSD__) && !defined(DOXYGEN))
+  // For VAAPI rate control types
+  #include <va/va.h>
+#endif
+
 namespace fs = std::filesystem;
 using namespace std::literals;
 
@@ -397,6 +402,89 @@ namespace config {
 
   }  // namespace qsv
 
+  namespace vaapi {
+#if !(defined(linux) || defined(__FreeBSD__) || defined(DOXYGEN))
+    constexpr int VA_RC_CBR = 0x00000002;
+    constexpr int VA_RC_VBR = 0x00000004;
+    constexpr int VA_RC_CQP = 0x00000010;
+    constexpr int VA_RC_ICQ = 0x00000040;
+    constexpr int VA_RC_QVBR = 0x00000400;
+    constexpr int VA_RC_AVBR = 0x00000800;
+#endif
+    /**
+     * @brief Enumerates supported VA-API quality options.
+     */
+    enum class quality_e : int {
+      _auto = 0,  ///< Auto quality level
+      speed = 1,  ///< Speed level
+      balanced = 2,  ///< Balanced level
+      quality = 3  ///< Quality level
+    };
+
+    /**
+     * @brief Enumerates supported VA-API rc options.
+     */
+    enum class rc_e : int {
+      _auto = 0,  ///< Auto rate control
+      avbr = VA_RC_AVBR,  ///< AVBR - average variable bitrate
+      cbr = VA_RC_CBR,  ///< CBR - constant bitrate
+      cqp = VA_RC_CQP,  ///< CQP - constant QP
+      icq = VA_RC_ICQ,  ///< ICQ - intelligent QP
+      qvbr = VA_RC_QVBR,  ///< QVBR - quality-defined variable bitrate
+      vbr = VA_RC_VBR  ///< VBR - variable bitrate
+    };
+
+    /**
+     * @brief Parse a VA-API quality preset while preserving the current value on invalid input.
+     *
+     * @param quality_type Configuration text naming the VA-API quality preset.
+     * @param original Original text value used when reporting a parsing failure.
+     * @return Parsed enum value, or the setting-specific default when the text is unknown.
+     */
+    template<class T>
+    ::std::optional<int> quality_from_view(const ::std::string_view &quality_type, const ::std::optional<int>(&original)) {
+#ifndef DOXYGEN
+  #define _CONVERT_(x) \
+    if (quality_type == #x##sv) \
+    return (int) T::x
+#endif
+      _CONVERT_(balanced);
+      _CONVERT_(quality);
+      _CONVERT_(speed);
+#ifdef _CONVERT_
+  #undef _CONVERT_
+#endif
+      return original;
+    }
+
+    /**
+     * @brief Parse a VA-API rate-control mode while preserving the current value on invalid input.
+     *
+     * @param rc Rate-control mode selected in the configuration.
+     * @param original Original text value used when reporting a parsing failure.
+     * @return Parsed enum value, or the setting-specific default when the text is unknown.
+     */
+    template<class T>
+    ::std::optional<int> rc_from_view(const ::std::string_view &rc, const ::std::optional<int>(&original)) {
+#ifndef DOXYGEN
+  #define _CONVERT_(x) \
+    if (rc == #x##sv) \
+    return (int) T::x
+#endif
+      _CONVERT_(avbr);
+      _CONVERT_(cbr);
+      _CONVERT_(cqp);
+      _CONVERT_(icq);
+      _CONVERT_(qvbr);
+      _CONVERT_(vbr);
+#ifdef _CONVERT_
+  #undef _CONVERT_
+#endif
+      return original;
+    }
+
+  }  // namespace vaapi
+
   namespace vt {
 
     /**
@@ -669,6 +757,10 @@ namespace config {
     },  // vt
 
     {
+      false,  // blbrc
+      std::to_underlying(vaapi::quality_e::_auto),  // quality
+      std::to_underlying(vaapi::rc_e::_auto),  // rate control
+      {},  // rate control string
       false,  // strict_rc_buffer
     },  // vaapi
 
@@ -1551,6 +1643,16 @@ namespace config {
     int_f(vars, "vt_software", video.vt.vt_require_sw, vt::force_software_from_view);
     int_f(vars, "vt_realtime", video.vt.vt_realtime, vt::rt_from_view);
 
+    std::string vaapi_quality;
+    string_f(vars, "vaapi_quality", vaapi_quality);
+    if (!vaapi_quality.empty()) {
+      video.vaapi.vaapi_quality = vaapi::quality_from_view<vaapi::quality_e>(vaapi_quality, video.vaapi.vaapi_quality);
+    }
+    string_f(vars, "vaapi_rc", video.vaapi.vaapi_rc_str);
+    if (!video.vaapi.vaapi_rc_str.empty()) {
+      video.vaapi.vaapi_rc = vaapi::rc_from_view<vaapi::rc_e>(video.vaapi.vaapi_rc_str, video.vaapi.vaapi_rc);
+    }
+    bool_f(vars, "vaapi_blbrc", video.vaapi.blbrc);
     bool_f(vars, "vaapi_strict_rc_buffer", video.vaapi.strict_rc_buffer);
 
     int_f(vars, "vk_tune", video.vk.tune);

src/config.h

@@ -105,6 +105,10 @@ namespace config {
     } vt;  ///< VideoToolbox encoder options.
 
     struct {
+      bool blbrc;
+      std::optional<int> vaapi_quality;
+      std::optional<int> vaapi_rc;
+      std::string vaapi_rc_str;
       bool strict_rc_buffer;
     } vaapi;  ///< VA-API encoder options.
 

src/platform/linux/vaapi.cpp

@@ -308,8 +308,33 @@ namespace va {
         BOOST_LOG(info) << "Using normal encoding mode"sv;
       }
 
+      // When the compression_level AVOption is set, vaapi_encode.c assigns the value to VAEncMiscParameterBufferQualityLevel
+      VAConfigAttrib quality_attr = {VAConfigAttribEncQualityRange};
+      auto status = vaGetConfigAttributes(va_display, va_profile, va_entrypoint, &quality_attr, 1);
+      if (status != VA_STATUS_SUCCESS || quality_attr.value == VA_ATTRIB_NOT_SUPPORTED) {
+        quality_attr.value = 0;
+      }
+      auto vaapi_quality = config::video.vaapi.vaapi_quality.value_or(0);
+      auto target_quality = 0;
+      switch (vaapi_quality) {
+        default:
+        case 0:  // auto or unset
+          break;
+        case 1:  // low quality (highest value in range)
+        case 2:  // med quality (middle value in range)
+          target_quality = quality_attr.value / vaapi_quality;
+          break;
+        case 3:  // high quality (1)
+          target_quality = 1;
+          break;
+      }
+      if (quality_attr.value > 0) {
+        ctx->compression_level = target_quality;
+        BOOST_LOG(info) << "[VAAPI] Quality level set to "sv << ctx->compression_level << " (fastest level: "sv << quality_attr.value << ")"sv;
+      }
+
       VAConfigAttrib rc_attr = {VAConfigAttribRateControl};
-      auto status = vaGetConfigAttributes(va_display, va_profile, va_entrypoint, &rc_attr, 1);
+      status = vaGetConfigAttributes(va_display, va_profile, va_entrypoint, &rc_attr, 1);
       if (status != VA_STATUS_SUCCESS) {
         // Stick to the default rate control (CQP)
         rc_attr.value = 0;
@@ -337,27 +362,52 @@ namespace va {
       // When we have to resort to the default 1 second VBV for encoding quality reasons,
       // we stick to CBR in order to avoid encoding huge frames after bitrate undershoots
       // leave headroom available in the RC window.
-      if (config::video.vaapi.strict_rc_buffer ||
-          (vendor && strstr(vendor, "Intel")) ||
-          ctx->codec_id == AV_CODEC_ID_AV1) {
+      //
+      // If a user-supplied rate control is detected, override the whitelist logic to allow
+      // full user control of both the rate control and strict VBV settings.
+      auto auto_whitelist = false;
+      auto rc_mode = config::video.vaapi.vaapi_rc_str;
+      auto rc_vbv = "with standard VBV size";
+      auto rc_whitelist = "";
+      auto rc_val = config::video.vaapi.vaapi_rc.value_or(0);
+
+      // Detect whitelisted configurations
+      if ((vendor && std::string_view(vendor).contains("Intel") == true) || ctx->codec_id == AV_CODEC_ID_AV1) {
+        auto_whitelist = true;
+        rc_whitelist = " (whitelist override)";
+      }
+
+      // First try user config, else fall back to auto-detection that respects whitelist
+      if (rc_val > 0 && rc_attr.value & rc_val) {
+        // override whitelist if the user-specified RC is supported
+        auto_whitelist = false;
+        rc_whitelist = "";
+      } else if (rc_attr.value & VA_RC_VBR && auto_whitelist) {
+        rc_mode = "vbr";
+        rc_val = VA_RC_VBR;
+      } else if (rc_attr.value & VA_RC_CBR) {
+        rc_mode = "cbr";
+        rc_val = VA_RC_CBR;
+      } else {
+        rc_mode = "cqp";
+        rc_val = VA_RC_CQP;
+      }
+
+      if (config::video.vaapi.strict_rc_buffer || auto_whitelist) {
         ctx->rc_buffer_size = ctx->bit_rate * ctx->framerate.den / ctx->framerate.num;
+        rc_vbv = "with single frame VBV size";
+      }
 
-        if (rc_attr.value & VA_RC_VBR) {
-          BOOST_LOG(info) << "Using VBR with single frame VBV size"sv;
-          av_dict_set(options, "rc_mode", "VBR", 0);
-        } else if (rc_attr.value & VA_RC_CBR) {
-          BOOST_LOG(info) << "Using CBR with single frame VBV size"sv;
-          av_dict_set(options, "rc_mode", "CBR", 0);
-        } else {
-          BOOST_LOG(warning) << "Using CQP with single frame VBV size"sv;
-          av_dict_set_int(options, "qp", config::video.qp, 0);
-        }
-      } else if (!(rc_attr.value & (VA_RC_CBR | VA_RC_VBR))) {
-        BOOST_LOG(warning) << "Using CQP rate control"sv;
+      // ffmpeg's rc_mode values don't align with VAAPI's rc_val values, so transform string to uppercase
+      std::transform(rc_mode.begin(), rc_mode.end(), rc_mode.begin(), [](unsigned char c) {
+        return std::toupper(c);
+      });
+      av_dict_set(options, "rc_mode", rc_mode.c_str(), 0);
+      if (rc_val == VA_RC_CQP || rc_val == VA_RC_ICQ || rc_val == VA_RC_QVBR) {
+        BOOST_LOG(warning) << "[VAAPI] Applying QP for compatible rate control method (QP value: "sv << config::video.qp << ")"sv;
         av_dict_set_int(options, "qp", config::video.qp, 0);
-      } else {
-        BOOST_LOG(info) << "Using default rate control"sv;
       }
+      BOOST_LOG(info) << "[VAAPI] Using "sv << rc_mode << " rate control "sv << rc_vbv << rc_whitelist;
     }
 
     /**