Make QualityScaler more responsive to downgrades.
Permits going from HD to QVGA in 6 seconds instead of 10. Also adds
windows for going up quickly in the beginning of a call (before any
downscaling happens due to bad quality).
BUG=webrtc:5678
R=glaznev@webrtc.org, stefan@webrtc.org
Review URL: https://codereview.webrtc.org/1830593003 .
Cr-Commit-Position: refs/heads/master@{#12219}
diff --git a/webrtc/api/java/jni/androidmediaencoder_jni.cc b/webrtc/api/java/jni/androidmediaencoder_jni.cc
index b46d82e..35291df 100644
--- a/webrtc/api/java/jni/androidmediaencoder_jni.cc
+++ b/webrtc/api/java/jni/androidmediaencoder_jni.cc
@@ -382,15 +382,17 @@
const int kLowQpThreshold = 32;
const int kBadQpThreshold = 92;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
- codec_settings->startBitrate,
- codec_settings->width, codec_settings->height);
+ codec_settings->startBitrate, codec_settings->width,
+ codec_settings->height,
+ codec_settings->maxFramerate);
} else if (codecType_ == kVideoCodecH264) {
// H264 QP is in the range [0, 51].
const int kLowQpThreshold = 21;
const int kBadQpThreshold = 36;
quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false,
- codec_settings->startBitrate,
- codec_settings->width, codec_settings->height);
+ codec_settings->startBitrate, codec_settings->width,
+ codec_settings->height,
+ codec_settings->maxFramerate);
} else {
// When adding codec support to additional hardware codecs, also configure
// their QP thresholds for scaling.
@@ -398,7 +400,6 @@
scale_ = false;
}
quality_scaler_.SetMinResolution(kMinDimension, kMinDimension);
- quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
init_width = std::max(res.width, kMinDimension);
init_height = std::max(res.height, kMinDimension);
diff --git a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
index a62bd41..b342886 100644
--- a/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
+++ b/webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
@@ -604,8 +604,8 @@
// TODO(glaznev/sprang): consider passing codec initial bitrate to quality
// scaler to avoid starting with HD for low initial bitrates.
quality_scaler_.Init(codec_.qpMax / QualityScaler::kDefaultLowQpDenominator,
- kDisabledBadQpThreshold, false, 0, 0, 0);
- quality_scaler_.ReportFramerate(codec_.maxFramerate);
+ kDisabledBadQpThreshold, false, 0, 0, 0,
+ codec_.maxFramerate);
// Only apply scaling to improve for single-layer streams. The scaling metrics
// use frame drops as a signal and is only applicable when we drop frames.
diff --git a/webrtc/modules/video_coding/utility/quality_scaler.cc b/webrtc/modules/video_coding/utility/quality_scaler.cc
index 68ae921..c6e5669 100644
--- a/webrtc/modules/video_coding/utility/quality_scaler.cc
+++ b/webrtc/modules/video_coding/utility/quality_scaler.cc
@@ -11,8 +11,11 @@
namespace webrtc {
-static const int kMinFps = 10;
-static const int kMeasureSeconds = 5;
+static const int kMinFps = 5;
+static const int kMeasureSecondsDownscale = 3;
+// Threshold constant used until first downscale (to permit fast rampup).
+static const int kMeasureSecondsFastUpscale = 2;
+static const int kMeasureSecondsUpscale = 5;
static const int kFramedropPercentThreshold = 60;
static const int kHdResolutionThreshold = 700 * 500;
static const int kHdBitrateThresholdKbps = 500;
@@ -23,9 +26,7 @@
const int QualityScaler::kDefaultMinDownscaleDimension = 90;
QualityScaler::QualityScaler()
- : num_samples_(0),
- low_qp_threshold_(-1),
- downscale_shift_(0),
+ : low_qp_threshold_(-1),
framerate_down_(false),
min_width_(kDefaultMinDownscaleDimension),
min_height_(kDefaultMinDownscaleDimension) {}
@@ -35,12 +36,17 @@
bool use_framerate_reduction,
int initial_bitrate_kbps,
int width,
- int height) {
+ int height,
+ int fps) {
ClearSamples();
low_qp_threshold_ = low_qp_threshold;
high_qp_threshold_ = high_qp_threshold;
use_framerate_reduction_ = use_framerate_reduction;
downscale_shift_ = 0;
+ // Use a faster window for upscaling initially (but be more graceful later).
+ // This enables faster initial rampups without risking strong up-down
+ // behavior later.
+ measure_seconds_upscale_ = kMeasureSecondsFastUpscale;
const int init_width = width;
const int init_height = height;
// TODO(glaznev): Investigate using thresholds for other resolutions
@@ -55,6 +61,7 @@
}
}
UpdateTargetResolution(init_width, init_height);
+ ReportFramerate(fps);
target_framerate_ = -1;
}
@@ -65,14 +72,14 @@
// Report framerate(fps) to estimate # of samples.
void QualityScaler::ReportFramerate(int framerate) {
- num_samples_ = static_cast<size_t>(
- kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
framerate_ = framerate;
+ UpdateSampleCounts();
}
void QualityScaler::ReportQP(int qp) {
framedrop_percent_.AddSample(0);
- average_qp_.AddSample(qp);
+ average_qp_downscale_.AddSample(qp);
+ average_qp_upscale_.AddSample(qp);
}
void QualityScaler::ReportDroppedFrame() {
@@ -82,7 +89,8 @@
void QualityScaler::OnEncodeFrame(const VideoFrame& frame) {
// Should be set through InitEncode -> Should be set by now.
assert(low_qp_threshold_ >= 0);
- assert(num_samples_ > 0);
+ assert(num_samples_upscale_ > 0);
+ assert(num_samples_downscale_ > 0);
// Update scale factor.
int avg_drop = 0;
@@ -90,9 +98,9 @@
// When encoder consistently overshoots, framerate reduction and spatial
// resizing will be triggered to get a smoother video.
- if ((framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
+ if ((framedrop_percent_.GetAverage(num_samples_downscale_, &avg_drop) &&
avg_drop >= kFramedropPercentThreshold) ||
- (average_qp_.GetAverage(num_samples_, &avg_qp) &&
+ (average_qp_downscale_.GetAverage(num_samples_downscale_, &avg_qp) &&
avg_qp > high_qp_threshold_)) {
// Reducing frame rate before spatial resolution change.
// Reduce frame rate only when it is above a certain number.
@@ -107,7 +115,7 @@
} else {
AdjustScale(false);
}
- } else if (average_qp_.GetAverage(num_samples_, &avg_qp) &&
+ } else if (average_qp_upscale_.GetAverage(num_samples_upscale_, &avg_qp) &&
avg_qp <= low_qp_threshold_) {
if (use_framerate_reduction_ && framerate_down_) {
target_framerate_ = -1;
@@ -160,13 +168,26 @@
void QualityScaler::ClearSamples() {
framedrop_percent_.Reset();
- average_qp_.Reset();
+ average_qp_downscale_.Reset();
+ average_qp_upscale_.Reset();
+}
+
+void QualityScaler::UpdateSampleCounts() {
+ num_samples_downscale_ = static_cast<size_t>(
+ kMeasureSecondsDownscale * (framerate_ < kMinFps ? kMinFps : framerate_));
+ num_samples_upscale_ = static_cast<size_t>(
+ measure_seconds_upscale_ * (framerate_ < kMinFps ? kMinFps : framerate_));
}
void QualityScaler::AdjustScale(bool up) {
downscale_shift_ += up ? -1 : 1;
if (downscale_shift_ < 0)
downscale_shift_ = 0;
+ if (!up) {
+ // Hit first downscale, start using a slower threshold for going up.
+ measure_seconds_upscale_ = kMeasureSecondsUpscale;
+ UpdateSampleCounts();
+ }
ClearSamples();
}
diff --git a/webrtc/modules/video_coding/utility/quality_scaler.h b/webrtc/modules/video_coding/utility/quality_scaler.h
index ebceb06..34dda0e 100644
--- a/webrtc/modules/video_coding/utility/quality_scaler.h
+++ b/webrtc/modules/video_coding/utility/quality_scaler.h
@@ -30,7 +30,8 @@
bool use_framerate_reduction,
int initial_bitrate_kbps,
int width,
- int height);
+ int height,
+ int fps);
void SetMinResolution(int min_width, int min_height);
void ReportFramerate(int framerate);
void ReportQP(int qp);
@@ -46,17 +47,22 @@
void AdjustScale(bool up);
void UpdateTargetResolution(int frame_width, int frame_height);
void ClearSamples();
+ void UpdateSampleCounts();
Scaler scaler_;
VideoFrame scaled_frame_;
- size_t num_samples_;
+ size_t num_samples_downscale_;
+ size_t num_samples_upscale_;
+ int measure_seconds_upscale_;
+ MovingAverage<int> average_qp_upscale_;
+ MovingAverage<int> average_qp_downscale_;
+
int framerate_;
int target_framerate_;
int low_qp_threshold_;
int high_qp_threshold_;
MovingAverage<int> framedrop_percent_;
- MovingAverage<int> average_qp_;
Resolution res_;
int downscale_shift_;
diff --git a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
index a5e3219..72e9db4 100644
--- a/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
+++ b/webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
@@ -27,6 +27,11 @@
static const int kMaxQp = 56;
static const int kDisabledBadQpThreshold = kMaxQp + 1;
static const int kLowInitialBitrateKbps = 300;
+// These values need to be in sync with corresponding constants
+// in quality_scaler.cc
+static const int kMeasureSecondsDownscale = 3;
+static const int kMeasureSecondsFastUpscale = 2;
+static const int kMeasureSecondsUpscale = 5;
} // namespace
class QualityScalerTest : public ::testing::Test {
@@ -51,8 +56,7 @@
input_frame_.CreateEmptyFrame(kWidth, kHeight, kWidth, kHalfWidth,
kHalfWidth);
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false,
- 0, 0, 0);
- qs_.ReportFramerate(kFramerate);
+ 0, 0, 0, kFramerate);
qs_.OnEncodeFrame(input_frame_);
}
@@ -104,7 +108,8 @@
int initial_framerate);
void VerifyQualityAdaptation(int initial_framerate,
- int seconds,
+ int seconds_downscale,
+ int seconds_upscale,
bool expect_spatial_resize,
bool expect_framerate_reduction);
@@ -298,18 +303,19 @@
void QualityScalerTest::VerifyQualityAdaptation(
int initial_framerate,
- int seconds,
+ int seconds_downscale,
+ int seconds_upscale,
bool expect_spatial_resize,
bool expect_framerate_reduction) {
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
- kDisabledBadQpThreshold, true, 0, 0, 0);
+ kDisabledBadQpThreshold, true, 0, 0, 0, initial_framerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
int init_height = qs_.GetScaledResolution().height;
// Test reducing framerate by dropping frame continuously.
QualityScalerTest::Resolution res =
- TriggerResolutionChange(kDropFrame, seconds, initial_framerate);
+ TriggerResolutionChange(kDropFrame, seconds_downscale, initial_framerate);
if (expect_framerate_reduction) {
EXPECT_LT(res.framerate, initial_framerate);
@@ -327,35 +333,40 @@
}
// The "seconds * 1.5" is to ensure spatial resolution to recover.
- // For example, in 10 seconds test, framerate reduction happens in the first
- // 5 seconds from 30fps to 15fps and causes the buffer size to be half of the
- // original one. Then it will take only 75 samples to downscale (twice in 150
+ // For example, in 6 seconds test, framerate reduction happens in the first
+ // 3 seconds from 30fps to 15fps and causes the buffer size to be half of the
+ // original one. Then it will take only 45 samples to downscale (twice in 90
// samples). So to recover the resolution changes, we need more than 10
- // seconds (i.e, seconds * 1.5). This is because the framerate increases
- // before spatial size recovers, so it will take 150 samples to recover
- // spatial size (300 for twice).
- res = TriggerResolutionChange(kReportLowQP, seconds * 1.5, initial_framerate);
+ // seconds (i.e, seconds_upscale * 1.5). This is because the framerate
+ // increases before spatial size recovers, so it will take 150 samples to
+ // recover spatial size (300 for twice).
+ res = TriggerResolutionChange(kReportLowQP, seconds_upscale * 1.5,
+ initial_framerate);
EXPECT_EQ(-1, res.framerate);
EXPECT_EQ(init_width, res.width);
EXPECT_EQ(init_height, res.height);
}
-// In 5 seconds test, only framerate adjusting should happen.
+// In 3 seconds test, only framerate adjusting should happen and 5 second
+// upscaling duration, only a framerate adjusting should happen.
TEST_F(QualityScalerTest, ChangeFramerateOnly) {
- VerifyQualityAdaptation(kFramerate, 5, false, true);
+ VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale,
+ kMeasureSecondsUpscale, false, true);
}
-// In 10 seconds test, framerate adjusting and scaling are both
+// In 6 seconds test, framerate adjusting and scaling are both
// triggered, it shows that scaling would happen after framerate
// adjusting.
TEST_F(QualityScalerTest, ChangeFramerateAndSpatialSize) {
- VerifyQualityAdaptation(kFramerate, 10, true, true);
+ VerifyQualityAdaptation(kFramerate, kMeasureSecondsDownscale * 2,
+ kMeasureSecondsUpscale * 2, true, true);
}
// When starting from a low framerate, only spatial size will be changed.
TEST_F(QualityScalerTest, ChangeSpatialSizeOnly) {
qs_.ReportFramerate(kFramerate >> 1);
- VerifyQualityAdaptation(kFramerate >> 1, 10, true, false);
+ VerifyQualityAdaptation(kFramerate >> 1, kMeasureSecondsDownscale * 2,
+ kMeasureSecondsUpscale * 2, true, false);
}
TEST_F(QualityScalerTest, DoesNotDownscaleBelow2xDefaultMinDimensionsWidth) {
@@ -371,7 +382,7 @@
TEST_F(QualityScalerTest, DownscaleToVgaOnLowInitialBitrate) {
qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator,
kDisabledBadQpThreshold, true,
- kLowInitialBitrateKbps, kWidth, kHeight);
+ kLowInitialBitrateKbps, kWidth, kHeight, kFramerate);
qs_.OnEncodeFrame(input_frame_);
int init_width = qs_.GetScaledResolution().width;
int init_height = qs_.GetScaledResolution().height;
@@ -379,6 +390,70 @@
EXPECT_LE(init_height, kHeightVga);
}
+TEST_F(QualityScalerTest, DownscaleAfterMeasuredSecondsThenSlowerBackUp) {
+ QualityScalerTest::Resolution initial_res;
+ qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false, 0,
+ kWidth, kHeight, kFramerate);
+ qs_.OnEncodeFrame(input_frame_);
+ initial_res.width = qs_.GetScaledResolution().width;
+ initial_res.height = qs_.GetScaledResolution().height;
+
+ // Should not downscale if less than kMeasureSecondsDownscale seconds passed.
+ for (int i = 0; i < kFramerate * kMeasureSecondsDownscale - 1; ++i) {
+ qs_.ReportQP(kHighQp + 1);
+ qs_.OnEncodeFrame(input_frame_);
+ }
+ EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
+ EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
+
+ // Should downscale if more than kMeasureSecondsDownscale seconds passed (add
+ // last frame).
+ qs_.ReportQP(kHighQp + 1);
+ qs_.OnEncodeFrame(input_frame_);
+ EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width);
+ EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height);
+
+ // Should not upscale if less than kMeasureSecondsUpscale seconds passed since
+ // we saw issues initially (have already gone down).
+ for (int i = 0; i < kFramerate * kMeasureSecondsUpscale - 1; ++i) {
+ qs_.ReportQP(kLowQp);
+ qs_.OnEncodeFrame(input_frame_);
+ }
+ EXPECT_GT(initial_res.width, qs_.GetScaledResolution().width);
+ EXPECT_GT(initial_res.height, qs_.GetScaledResolution().height);
+
+ // Should upscale (back to initial) if kMeasureSecondsUpscale seconds passed
+ // (add last frame).
+ qs_.ReportQP(kLowQp);
+ qs_.OnEncodeFrame(input_frame_);
+ EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
+ EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
+}
+
+TEST_F(QualityScalerTest, UpscaleQuicklyInitiallyAfterMeasuredSeconds) {
+ QualityScalerTest::Resolution initial_res;
+ qs_.Init(kMaxQp / QualityScaler::kDefaultLowQpDenominator, kHighQp, false,
+ kLowInitialBitrateKbps, kWidth, kHeight, kFramerate);
+ qs_.OnEncodeFrame(input_frame_);
+ initial_res.width = qs_.GetScaledResolution().width;
+ initial_res.height = qs_.GetScaledResolution().height;
+
+ // Should not upscale if less than kMeasureSecondsFastUpscale seconds passed.
+ for (int i = 0; i < kFramerate * kMeasureSecondsFastUpscale - 1; ++i) {
+ qs_.ReportQP(kLowQp);
+ qs_.OnEncodeFrame(input_frame_);
+ }
+ EXPECT_EQ(initial_res.width, qs_.GetScaledResolution().width);
+ EXPECT_EQ(initial_res.height, qs_.GetScaledResolution().height);
+
+ // Should upscale if kMeasureSecondsFastUpscale seconds passed (add last
+ // frame).
+ qs_.ReportQP(kLowQp);
+ qs_.OnEncodeFrame(input_frame_);
+ EXPECT_LT(initial_res.width, qs_.GetScaledResolution().width);
+ EXPECT_LT(initial_res.height, qs_.GetScaledResolution().height);
+}
+
void QualityScalerTest::DownscaleEndsAt(int input_width,
int input_height,
int end_width,