Add ability to specify if rate controller of video encoder is trusted.
If rate controller is trusted, we disable the frame dropper in the
media optimization module.
This is a re-land of
https://webrtc-review.googlesource.com/c/src/+/105020
Bug: webrtc:9890
Change-Id: I418e47a43a1a98cb2fd5295c03360b954f2288f2
Reviewed-on: https://webrtc-review.googlesource.com/c/109141
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25570}diff --git a/api/video_codecs/test/BUILD.gn b/api/video_codecs/test/BUILD.gn
index fa19162..6f9dcaa 100644
--- a/api/video_codecs/test/BUILD.gn
+++ b/api/video_codecs/test/BUILD.gn
@@ -21,6 +21,7 @@
"..:builtin_video_encoder_factory",
"..:rtc_software_fallback_wrappers",
"..:video_codecs_api",
+ "../..:mock_video_encoder",
"../../../modules/video_coding:video_codec_interface",
"../../../modules/video_coding:video_coding_utility",
"../../../modules/video_coding:webrtc_vp8",
diff --git a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
index 555914e..b374f3d 100644
--- a/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
+++ b/api/video_codecs/test/video_encoder_software_fallback_wrapper_unittest.cc
@@ -10,6 +10,7 @@
#include <utility>
+#include "api/test/mock_video_encoder.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_bitrate_allocation.h"
#include "api/video_codecs/video_encoder_software_fallback_wrapper.h"
@@ -21,9 +22,12 @@
#include "rtc_base/checks.h"
#include "rtc_base/fakeclock.h"
#include "test/field_trial.h"
+#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
+using ::testing::Return;
+
namespace {
const int kWidth = 320;
const int kHeight = 240;
@@ -33,6 +37,12 @@
const int kDefaultMinPixelsPerFrame = 320 * 180;
const int kLowThreshold = 10;
const int kHighThreshold = 20;
+
+VideoEncoder::EncoderInfo GetEncoderInfo(bool trusted_rate_controller) {
+ VideoEncoder::EncoderInfo info;
+ info.has_trusted_rate_controller = trusted_rate_controller;
+ return info;
+}
} // namespace
class VideoEncoderSoftwareFallbackWrapperTest : public ::testing::Test {
@@ -512,4 +522,73 @@
EXPECT_FALSE(settings.thresholds.has_value());
}
+TEST(SoftwareFallbackEncoderTest, BothRateControllersNotTrusted) {
+ auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+
+ EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(false)));
+ EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(false)));
+
+ std::unique_ptr<VideoEncoder> wrapper =
+ CreateVideoEncoderSoftwareFallbackWrapper(
+ std::unique_ptr<VideoEncoder>(sw_encoder),
+ std::unique_ptr<VideoEncoder>(hw_encoder));
+ EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
+TEST(SoftwareFallbackEncoderTest, SwRateControllerTrusted) {
+ auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(true)));
+ EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(false)));
+
+ std::unique_ptr<VideoEncoder> wrapper =
+ CreateVideoEncoderSoftwareFallbackWrapper(
+ std::unique_ptr<VideoEncoder>(sw_encoder),
+ std::unique_ptr<VideoEncoder>(hw_encoder));
+ EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
+TEST(SoftwareFallbackEncoderTest, HwRateControllerTrusted) {
+ auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(false)));
+ EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(true)));
+
+ std::unique_ptr<VideoEncoder> wrapper =
+ CreateVideoEncoderSoftwareFallbackWrapper(
+ std::unique_ptr<VideoEncoder>(sw_encoder),
+ std::unique_ptr<VideoEncoder>(hw_encoder));
+ EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+
+ // Trigger fallback to software.
+ EXPECT_CALL(*hw_encoder, Encode)
+ .WillOnce(Return(WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE));
+ VideoFrame frame = VideoFrame::Builder().build();
+ wrapper->Encode(frame, nullptr, nullptr);
+
+ EXPECT_FALSE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
+TEST(SoftwareFallbackEncoderTest, BothRateControllersTrusted) {
+ auto* sw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ auto* hw_encoder = new testing::NiceMock<MockVideoEncoder>();
+ EXPECT_CALL(*sw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(true)));
+ EXPECT_CALL(*hw_encoder, GetEncoderInfo())
+ .WillRepeatedly(Return(GetEncoderInfo(true)));
+
+ std::unique_ptr<VideoEncoder> wrapper =
+ CreateVideoEncoderSoftwareFallbackWrapper(
+ std::unique_ptr<VideoEncoder>(sw_encoder),
+ std::unique_ptr<VideoEncoder>(hw_encoder));
+ EXPECT_TRUE(wrapper->GetEncoderInfo().has_trusted_rate_controller);
+}
+
} // namespace webrtc
diff --git a/api/video_codecs/video_encoder.cc b/api/video_codecs/video_encoder.cc
index 0963886..363edbf 100644
--- a/api/video_codecs/video_encoder.cc
+++ b/api/video_codecs/video_encoder.cc
@@ -86,7 +86,8 @@
VideoEncoder::EncoderInfo::EncoderInfo()
: scaling_settings(VideoEncoder::ScalingSettings::kOff),
supports_native_handle(false),
- implementation_name("unknown") {}
+ implementation_name("unknown"),
+ has_trusted_rate_controller(false) {}
VideoEncoder::EncoderInfo::~EncoderInfo() = default;
diff --git a/api/video_codecs/video_encoder.h b/api/video_codecs/video_encoder.h
index f8f0313..d881f8a 100644
--- a/api/video_codecs/video_encoder.h
+++ b/api/video_codecs/video_encoder.h
@@ -132,6 +132,19 @@
// The name of this particular encoder implementation, e.g. "libvpx".
std::string implementation_name;
+
+ // If this field is true, the encoder rate controller must perform
+ // well even in difficult situations, and produce close to the specified
+ // target bitrate seen over a reasonable time window, drop frames if
+ // necessary in order to keep the rate correct, and react quickly to
+ // changing bitrate targets. If this method returns true, we disable the
+ // frame dropper in the media optimization module and rely entirely on the
+ // encoder to produce media at a bitrate that closely matches the target.
+ // Any overshooting may result in delay buildup. If this method returns
+ // false (default behavior), the media opt frame dropper will drop input
+ // frames if it suspect encoder misbehavior. Misbehavior is common,
+ // especially in hardware codecs. Disable media opt at your own risk.
+ bool has_trusted_rate_controller;
};
static VideoCodecVP8 GetDefaultVp8Settings();
@@ -220,6 +233,10 @@
virtual bool SupportsNativeHandle() const;
virtual const char* ImplementationName() const;
+ // Returns meta-data about the encoder, such as implementation name.
+ // The output of this method may change during runtime. For instance if a
+ // hardware encoder fails, it may fall back to doing software encoding using
+ // an implementation with different characteristics.
virtual EncoderInfo GetEncoderInfo() const;
};
} // namespace webrtc