Make quality scaler downscale faster.
Include dropped frames by the encoder in the frame drop percentage.
To react faster at low framerates:
- Use ExpFilter instead of MovingAverage to filter QP values.
- Reduce sampling interval while waiting for minimum number of needed frames (when not in fast rampup mode).
A separate slower ExpFilter is used for upscaling.
Bug: webrtc:9169
Change-Id: If7ff6c3bd4201fda2da67125889838fe96ce7061
Reviewed-on: https://webrtc-review.googlesource.com/70761
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23014}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 619f15e..b2fd91d 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -243,6 +243,7 @@
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
"../../rtc_base:sequenced_task_checker",
+ "../../rtc_base/experiments:quality_scaling_experiment",
"../../system_wrappers",
"../rtp_rtcp:rtp_rtcp_format",
]
diff --git a/modules/video_coding/utility/quality_scaler.cc b/modules/video_coding/utility/quality_scaler.cc
index e839f9e..273d7a6 100644
--- a/modules/video_coding/utility/quality_scaler.cc
+++ b/modules/video_coding/utility/quality_scaler.cc
@@ -17,7 +17,9 @@
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
+#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/task_queue.h"
+#include "rtc_base/timeutils.h"
// TODO(kthelgason): Some versions of Android have issues with log2.
// See https://code.google.com/p/android/issues/detail?id=212634 for details
@@ -37,6 +39,33 @@
} // namespace
+class QualityScaler::QpSmoother {
+ public:
+ explicit QpSmoother(float alpha)
+ : alpha_(alpha), last_sample_ms_(rtc::TimeMillis()), smoother_(alpha) {}
+
+ rtc::Optional<int> GetAvg() const {
+ float value = smoother_.filtered();
+ if (value == rtc::ExpFilter::kValueUndefined) {
+ return rtc::nullopt;
+ }
+ return static_cast<int>(value);
+ }
+
+ void Add(float sample) {
+ int64_t now_ms = rtc::TimeMillis();
+ smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
+ last_sample_ms_ = now_ms;
+ }
+
+ void Reset() { smoother_.Reset(alpha_); }
+
+ private:
+ const float alpha_;
+ int64_t last_sample_ms_;
+ rtc::ExpFilter smoother_;
+};
+
class QualityScaler::CheckQpTask : public rtc::QueuedTask {
public:
explicit CheckQpTask(QualityScaler* scaler) : scaler_(scaler) {
@@ -81,8 +110,16 @@
fast_rampup_(true),
// Arbitrarily choose size based on 30 fps for 5 seconds.
average_qp_(5 * 30),
- framedrop_percent_(5 * 30) {
+ framedrop_percent_media_opt_(5 * 30),
+ framedrop_percent_all_(5 * 30),
+ experiment_enabled_(QualityScalingExperiment::Enabled()),
+ observed_enough_frames_(false) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
+ if (experiment_enabled_) {
+ config_ = QualityScalingExperiment::GetConfig();
+ qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
+ qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
+ }
RTC_DCHECK(observer_ != nullptr);
check_qp_task_ = new CheckQpTask(this);
RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
@@ -96,19 +133,36 @@
int64_t QualityScaler::GetSamplingPeriodMs() const {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- return fast_rampup_ ? sampling_period_ms_
- : (sampling_period_ms_ * kSamplePeriodScaleFactor);
+ if (fast_rampup_) {
+ return sampling_period_ms_;
+ }
+ if (experiment_enabled_ && !observed_enough_frames_) {
+ // Use half the interval while waiting for enough frames.
+ return sampling_period_ms_ / 2;
+ }
+ return sampling_period_ms_ * kSamplePeriodScaleFactor;
}
-void QualityScaler::ReportDroppedFrame() {
+void QualityScaler::ReportDroppedFrameByMediaOpt() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- framedrop_percent_.AddSample(100);
+ framedrop_percent_media_opt_.AddSample(100);
+ framedrop_percent_all_.AddSample(100);
+}
+
+void QualityScaler::ReportDroppedFrameByEncoder() {
+ RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
+ framedrop_percent_all_.AddSample(100);
}
void QualityScaler::ReportQp(int qp) {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- framedrop_percent_.AddSample(0);
+ framedrop_percent_media_opt_.AddSample(0);
+ framedrop_percent_all_.AddSample(0);
average_qp_.AddSample(qp);
+ if (qp_smoother_high_)
+ qp_smoother_high_->Add(qp);
+ if (qp_smoother_low_)
+ qp_smoother_low_->Add(qp);
}
void QualityScaler::CheckQp() {
@@ -118,11 +172,19 @@
// If we have not observed at least this many frames we can't make a good
// scaling decision.
- if (framedrop_percent_.size() < kMinFramesNeededToScale)
+ const size_t frames = config_.use_all_drop_reasons
+ ? framedrop_percent_all_.size()
+ : framedrop_percent_media_opt_.size();
+ if (frames < kMinFramesNeededToScale) {
+ observed_enough_frames_ = false;
return;
+ }
+ observed_enough_frames_ = true;
// Check if we should scale down due to high frame drop.
- const rtc::Optional<int> drop_rate = framedrop_percent_.GetAverage();
+ const rtc::Optional<int> drop_rate =
+ config_.use_all_drop_reasons ? framedrop_percent_all_.GetAverage()
+ : framedrop_percent_media_opt_.GetAverage();
if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
ReportQpHigh();
@@ -130,14 +192,19 @@
}
// Check if we should scale up or down based on QP.
- const rtc::Optional<int> avg_qp = average_qp_.GetAverage();
- if (avg_qp) {
- RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp;
- if (*avg_qp > thresholds_.high) {
+ const rtc::Optional<int> avg_qp_high = qp_smoother_high_
+ ? qp_smoother_high_->GetAvg()
+ : average_qp_.GetAverage();
+ const rtc::Optional<int> avg_qp_low =
+ qp_smoother_low_ ? qp_smoother_low_->GetAvg() : average_qp_.GetAverage();
+ if (avg_qp_high && avg_qp_low) {
+ RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
+ << *avg_qp_low << ").";
+ if (*avg_qp_high > thresholds_.high) {
ReportQpHigh();
return;
}
- if (*avg_qp <= thresholds_.low) {
+ if (*avg_qp_low <= thresholds_.low) {
// QP has been low. We want to try a higher resolution.
ReportQpLow();
return;
@@ -163,7 +230,12 @@
void QualityScaler::ClearSamples() {
RTC_DCHECK_CALLED_SEQUENTIALLY(&task_checker_);
- framedrop_percent_.Reset();
+ framedrop_percent_media_opt_.Reset();
+ framedrop_percent_all_.Reset();
average_qp_.Reset();
+ if (qp_smoother_high_)
+ qp_smoother_high_->Reset();
+ if (qp_smoother_low_)
+ qp_smoother_low_->Reset();
}
} // namespace webrtc
diff --git a/modules/video_coding/utility/quality_scaler.h b/modules/video_coding/utility/quality_scaler.h
index e823461..996b9f2 100644
--- a/modules/video_coding/utility/quality_scaler.h
+++ b/modules/video_coding/utility/quality_scaler.h
@@ -11,12 +11,14 @@
#ifndef MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
#define MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
+#include <memory>
#include <utility>
#include "api/optional.h"
#include "api/video_codecs/video_encoder.h"
#include "common_types.h" // NOLINT(build/include)
#include "modules/video_coding/utility/moving_average.h"
+#include "rtc_base/experiments/quality_scaling_experiment.h"
#include "rtc_base/sequenced_task_checker.h"
namespace webrtc {
@@ -49,8 +51,9 @@
QualityScaler(AdaptationObserverInterface* observer,
VideoEncoder::QpThresholds thresholds);
virtual ~QualityScaler();
- // Should be called each time the encoder drops a frame.
- void ReportDroppedFrame();
+ // Should be called each time a frame is dropped at encoding.
+ void ReportDroppedFrameByMediaOpt();
+ void ReportDroppedFrameByEncoder();
// Inform the QualityScaler of the last seen QP.
void ReportQp(int qp);
@@ -62,6 +65,7 @@
private:
class CheckQpTask;
+ class QpSmoother;
void CheckQp();
void ClearSamples();
void ReportQpLow();
@@ -76,7 +80,15 @@
const int64_t sampling_period_ms_;
bool fast_rampup_ RTC_GUARDED_BY(&task_checker_);
MovingAverage average_qp_ RTC_GUARDED_BY(&task_checker_);
- MovingAverage framedrop_percent_ RTC_GUARDED_BY(&task_checker_);
+ MovingAverage framedrop_percent_media_opt_ RTC_GUARDED_BY(&task_checker_);
+ MovingAverage framedrop_percent_all_ RTC_GUARDED_BY(&task_checker_);
+
+ // Used by QualityScalingExperiment.
+ const bool experiment_enabled_;
+ QualityScalingExperiment::Config config_ RTC_GUARDED_BY(&task_checker_);
+ std::unique_ptr<QpSmoother> qp_smoother_high_ RTC_GUARDED_BY(&task_checker_);
+ std::unique_ptr<QpSmoother> qp_smoother_low_ RTC_GUARDED_BY(&task_checker_);
+ bool observed_enough_frames_ RTC_GUARDED_BY(&task_checker_);
};
} // namespace webrtc
diff --git a/modules/video_coding/utility/quality_scaler_unittest.cc b/modules/video_coding/utility/quality_scaler_unittest.cc
index 824826d..58a381c 100644
--- a/modules/video_coding/utility/quality_scaler_unittest.cc
+++ b/modules/video_coding/utility/quality_scaler_unittest.cc
@@ -11,9 +11,11 @@
#include "modules/video_coding/utility/quality_scaler.h"
#include <memory>
+#include <string>
#include "rtc_base/event.h"
#include "rtc_base/task_queue.h"
+#include "test/field_trial.h"
#include "test/gmock.h"
#include "test/gtest.h"
@@ -63,7 +65,8 @@
: QualityScaler(observer, thresholds, 5) {}
};
-class QualityScalerTest : public ::testing::Test {
+class QualityScalerTest : public ::testing::Test,
+ public ::testing::WithParamInterface<std::string> {
protected:
enum ScaleDirection {
kKeepScaleAboveLowQp,
@@ -74,7 +77,8 @@
};
QualityScalerTest()
- : q_(new rtc::TaskQueue("QualityScalerTestQueue")),
+ : scoped_field_trial_(GetParam()),
+ q_(new rtc::TaskQueue("QualityScalerTestQueue")),
observer_(new MockAdaptationObserver()) {
DO_SYNC(q_, {
qs_ = std::unique_ptr<QualityScaler>(new QualityScalerUnderTest(
@@ -96,7 +100,7 @@
qs_->ReportQp(kLowQp);
break;
case kScaleDown:
- qs_->ReportDroppedFrame();
+ qs_->ReportDroppedFrameByMediaOpt();
break;
case kKeepScaleAtHighQp:
qs_->ReportQp(kHighQp);
@@ -108,37 +112,45 @@
}
}
+ test::ScopedFieldTrials scoped_field_trial_;
std::unique_ptr<rtc::TaskQueue> q_;
std::unique_ptr<QualityScaler> qs_;
std::unique_ptr<MockAdaptationObserver> observer_;
};
-TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
+INSTANTIATE_TEST_CASE_P(
+ FieldTrials,
+ QualityScalerTest,
+ ::testing::Values(
+ "WebRTC-Video-QualityScaling/Enabled-1,2,3,4,5,6,7,8,0.9,0.99,1/",
+ ""));
+
+TEST_P(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
DO_SYNC(q_, { TriggerScale(kScaleDown); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(1, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, KeepsScaleAtHighQp) {
+TEST_P(QualityScalerTest, KeepsScaleAtHighQp) {
DO_SYNC(q_, { TriggerScale(kKeepScaleAtHighQp); });
EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(0, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, DownscalesAboveHighQp) {
+TEST_P(QualityScalerTest, DownscalesAboveHighQp) {
DO_SYNC(q_, { TriggerScale(kScaleDownAboveHighQp); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(1, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
+TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
DO_SYNC(q_, {
for (int i = 0; i < kFramerate * 5; ++i) {
- qs_->ReportDroppedFrame();
- qs_->ReportDroppedFrame();
+ qs_->ReportDroppedFrameByMediaOpt();
+ qs_->ReportDroppedFrameByMediaOpt();
qs_->ReportQp(kHighQp);
}
});
@@ -147,10 +159,10 @@
EXPECT_EQ(0, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
+TEST_P(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
DO_SYNC(q_, {
for (int i = 0; i < kFramerate * 5; ++i) {
- qs_->ReportDroppedFrame();
+ qs_->ReportDroppedFrameByMediaOpt();
qs_->ReportQp(kHighQp);
}
});
@@ -159,21 +171,35 @@
EXPECT_EQ(0, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, KeepsScaleOnNormalQp) {
+TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsIfFieldTrialEnabled) {
+ const bool kDownScaleExpected = !GetParam().empty();
+ DO_SYNC(q_, {
+ for (int i = 0; i < kFramerate * 5; ++i) {
+ qs_->ReportDroppedFrameByMediaOpt();
+ qs_->ReportDroppedFrameByEncoder();
+ qs_->ReportQp(kHighQp);
+ }
+ });
+ EXPECT_EQ(kDownScaleExpected, observer_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(kDownScaleExpected ? 1 : 0, observer_->adapt_down_events_);
+ EXPECT_EQ(0, observer_->adapt_up_events_);
+}
+
+TEST_P(QualityScalerTest, KeepsScaleOnNormalQp) {
DO_SYNC(q_, { TriggerScale(kKeepScaleAboveLowQp); });
EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(0, observer_->adapt_down_events_);
EXPECT_EQ(0, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, UpscalesAfterLowQp) {
+TEST_P(QualityScalerTest, UpscalesAfterLowQp) {
DO_SYNC(q_, { TriggerScale(kScaleUp); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(0, observer_->adapt_down_events_);
EXPECT_EQ(1, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, ScalesDownAndBackUp) {
+TEST_P(QualityScalerTest, ScalesDownAndBackUp) {
DO_SYNC(q_, { TriggerScale(kScaleDown); });
EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
EXPECT_EQ(1, observer_->adapt_down_events_);
@@ -184,7 +210,7 @@
EXPECT_EQ(1, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) {
+TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) {
DO_SYNC(q_, {
// Not enough frames to make a decision.
for (int i = 0; i < kMinFramesNeededToScale - 1; ++i) {
@@ -210,7 +236,7 @@
EXPECT_EQ(1, observer_->adapt_up_events_);
}
-TEST_F(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) {
+TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) {
DO_SYNC(q_, {
for (int i = 0; i < kMinFramesNeededToScale; ++i) {
qs_->ReportQp(kHighQp + 1);