Asynchronous QualityScaler: Callback-based CheckQpTask.
This CL breaks up the CheckQp() operation into several steps managed
by the inner helper class CheckQpTask, making responding to high or
low QP an asynchronous operation. Why? Reconfiguring the stream in
response to QP overuse will in the future be handled on a separate
task queue. See Call-Level Adaptation Processing for more details:
https://docs.google.com/document/d/1ZyC26yOCknrrcYa839ZWLxD6o6Gig5A3lVTh4E41074/edit?usp=sharing
Instead of "bool AdaptDown()" when high QP is reported,
synchronously returning true or false depending on the result of
adaptation, this CL introduces
void QualityScalerQpUsageHandlerInterface::OnReportQpUsageHigh(
rtc::scoped_refptr<QualityScalerQpUsageHandlerCallback>);
Where
QualityScalerQpUsageHandlerCallback::OnQpUsageHandled(
bool clear_qp_samples);
Instructs the QualityScaler whether to clear samples before
checking QP the next time or to increase the frequency of checking
(corresponding to AdaptDown's return value prior to this CL).
QualityScaler no longer using AdaptationObserverInterface, this class
is renamed and moved to overuse_frame_detector.h.
The dependency between CheckQpTasks is made explicit with
CheckQpTask::Result and variables like observed_enough_frames_,
adapt_called_ and adapt_failed_ are moved there and given more
descriptive names.
Bug: webrtc:11521
Change-Id: I7faf795aeee5ded18ce75eb1617f88226e337228
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/173760
Reviewed-by: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Commit-Queue: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31140}
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 364aa0e..b143839 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -87,6 +87,7 @@
"../../api/video:video_bitrate_allocation",
"../../api/video:video_bitrate_allocator_factory",
"../../rtc_base:deprecation",
+ "../../rtc_base/task_utils:to_queued_task",
"../../system_wrappers:field_trial",
"../../system_wrappers:metrics",
"../rtp_rtcp:rtp_video_header",
@@ -303,6 +304,7 @@
deps = [
":video_codec_interface",
"..:module_api",
+ "../../api:scoped_refptr",
"../../api/video:encoded_frame",
"../../api/video:encoded_image",
"../../api/video:video_adaptation",
@@ -315,6 +317,7 @@
"../../rtc_base:rtc_base_approved",
"../../rtc_base:rtc_numerics",
"../../rtc_base:rtc_task_queue",
+ "../../rtc_base:weak_ptr",
"../../rtc_base/experiments:quality_scaler_settings",
"../../rtc_base/experiments:quality_scaling_experiment",
"../../rtc_base/experiments:rate_control_settings",
@@ -323,6 +326,7 @@
"../../rtc_base/system:arch",
"../../rtc_base/system:file_wrapper",
"../../rtc_base/task_utils:repeating_task",
+ "../../rtc_base/task_utils:to_queued_task",
"../../system_wrappers:field_trial",
"../rtp_rtcp:rtp_rtcp_format",
"//third_party/abseil-cpp/absl/types:optional",
diff --git a/modules/video_coding/utility/quality_scaler.cc b/modules/video_coding/utility/quality_scaler.cc
index c3d8b2e..e909b2f 100644
--- a/modules/video_coding/utility/quality_scaler.cc
+++ b/modules/video_coding/utility/quality_scaler.cc
@@ -19,6 +19,8 @@
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/task_queue.h"
+#include "rtc_base/task_utils/to_queued_task.h"
+#include "rtc_base/weak_ptr.h"
// TODO(kthelgason): Some versions of Android have issues with log2.
// See https://code.google.com/p/android/issues/detail?id=212634 for details
@@ -69,15 +71,192 @@
rtc::ExpFilter smoother_;
};
-QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
+// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
+// task will either run to completion and trigger a new task being queued, or it
+// will be destroyed because the QualityScaler is destroyed.
+//
+// When high or low QP is reported, the task will be pending until a callback is
+// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
+// asynchronously and prevents checking for QP until the stream has potentially
+// been reconfigured.
+class QualityScaler::CheckQpTask {
+ public:
+ // The result of one CheckQpTask may influence the delay of the next
+ // CheckQpTask.
+ struct Result {
+ bool observed_enough_frames = false;
+ bool qp_usage_reported = false;
+ bool clear_qp_samples = false;
+ };
+
+ CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
+ : quality_scaler_(quality_scaler),
+ state_(State::kNotStarted),
+ previous_task_result_(previous_task_result),
+ weak_ptr_factory_(this) {}
+
+ void StartDelayedTask() {
+ RTC_DCHECK_EQ(state_, State::kNotStarted);
+ state_ = State::kCheckingQp;
+ TaskQueueBase::Current()->PostDelayedTask(
+ ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
+ if (!this_weak_ptr) {
+ // The task has been cancelled through destruction.
+ return;
+ }
+ RTC_DCHECK_EQ(state_, State::kCheckingQp);
+ RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
+ switch (quality_scaler_->CheckQp()) {
+ case QualityScaler::CheckQpResult::kInsufficientSamples: {
+ result_.observed_enough_frames = false;
+ // After this line, |this| may be deleted.
+ DoCompleteTask();
+ return;
+ }
+ case QualityScaler::CheckQpResult::kNormalQp: {
+ result_.observed_enough_frames = true;
+ // After this line, |this| may be deleted.
+ DoCompleteTask();
+ return;
+ }
+ case QualityScaler::CheckQpResult::kHighQp: {
+ result_.observed_enough_frames = true;
+ result_.qp_usage_reported = true;
+ state_ = State::kAwaitingQpUsageHandled;
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
+ callback = ConstructCallback();
+ quality_scaler_->fast_rampup_ = false;
+ // After this line, |this| may be deleted.
+ quality_scaler_->handler_->OnReportQpUsageHigh(callback);
+ return;
+ }
+ case QualityScaler::CheckQpResult::kLowQp: {
+ result_.observed_enough_frames = true;
+ result_.qp_usage_reported = true;
+ state_ = State::kAwaitingQpUsageHandled;
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
+ callback = ConstructCallback();
+ // After this line, |this| may be deleted.
+ quality_scaler_->handler_->OnReportQpUsageLow(callback);
+ return;
+ }
+ }
+ }),
+ GetCheckingQpDelayMs());
+ }
+
+ void OnQpUsageHandled(bool clear_qp_samples) {
+ RTC_DCHECK_EQ(state_, State::kAwaitingQpUsageHandled);
+ result_.clear_qp_samples = clear_qp_samples;
+ if (clear_qp_samples)
+ quality_scaler_->ClearSamples();
+ DoCompleteTask();
+ }
+
+ bool HasCompletedTask() const { return state_ == State::kCompleted; }
+
+ Result result() const {
+ RTC_DCHECK(HasCompletedTask());
+ return result_;
+ }
+
+ private:
+ enum class State {
+ kNotStarted,
+ kCheckingQp,
+ kAwaitingQpUsageHandled,
+ kCompleted,
+ };
+
+ // Defined after the definition of QualityScaler::CheckQpTaskHandlerCallback.
+ // Gets around a forward declaration issue.
+ rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
+ ConstructCallback();
+
+ // Determines the sampling period of CheckQpTasks.
+ int64_t GetCheckingQpDelayMs() const {
+ RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
+ if (quality_scaler_->fast_rampup_) {
+ return quality_scaler_->sampling_period_ms_;
+ }
+ if (quality_scaler_->experiment_enabled_ &&
+ !previous_task_result_.observed_enough_frames) {
+ // Use half the interval while waiting for enough frames.
+ return quality_scaler_->sampling_period_ms_ / 2;
+ }
+ if (!previous_task_result_.clear_qp_samples) {
+ // Check shortly again.
+ return quality_scaler_->sampling_period_ms_ / 8;
+ }
+ if (quality_scaler_->scale_factor_ &&
+ !previous_task_result_.qp_usage_reported) {
+ // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
+ return quality_scaler_->sampling_period_ms_ *
+ quality_scaler_->scale_factor_.value();
+ }
+ return quality_scaler_->sampling_period_ms_ *
+ quality_scaler_->initial_scale_factor_;
+ }
+
+ void DoCompleteTask() {
+ RTC_DCHECK(state_ == State::kCheckingQp ||
+ state_ == State::kAwaitingQpUsageHandled);
+ state_ = State::kCompleted;
+ // Starting the next task deletes the pending task. After this line, |this|
+ // has been deleted.
+ quality_scaler_->StartNextCheckQpTask();
+ }
+
+ QualityScaler* const quality_scaler_;
+ State state_;
+ const Result previous_task_result_;
+ Result result_;
+
+ rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
+};
+
+class QualityScaler::CheckQpTaskHandlerCallback
+ : public QualityScalerQpUsageHandlerCallbackInterface {
+ public:
+ CheckQpTaskHandlerCallback(
+ rtc::WeakPtr<QualityScaler::CheckQpTask> check_qp_task)
+ : QualityScalerQpUsageHandlerCallbackInterface(),
+ check_qp_task_(std::move(check_qp_task)),
+ was_handled_(false) {}
+
+ ~CheckQpTaskHandlerCallback() { RTC_DCHECK(was_handled_); }
+
+ void OnQpUsageHandled(bool clear_qp_samples) {
+ RTC_DCHECK(!was_handled_);
+ was_handled_ = true;
+ if (!check_qp_task_) {
+ // The task has been cancelled through destruction; the result of the
+ // operation is ignored.
+ return;
+ }
+ check_qp_task_->OnQpUsageHandled(clear_qp_samples);
+ }
+
+ private:
+ // The callback may outlive the QualityScaler and its task.
+ rtc::WeakPtr<QualityScaler::CheckQpTask> const check_qp_task_;
+ bool was_handled_;
+};
+
+rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
+QualityScaler::CheckQpTask::ConstructCallback() {
+ return new CheckQpTaskHandlerCallback(weak_ptr_factory_.GetWeakPtr());
+}
+
+QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds)
- : QualityScaler(observer, thresholds, kMeasureMs) {}
+ : QualityScaler(handler, thresholds, kMeasureMs) {}
// Protected ctor, should not be called directly.
-QualityScaler::QualityScaler(AdaptationObserverInterface* observer,
+QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds,
int64_t sampling_period_ms)
- : observer_(observer),
+ : handler_(handler),
thresholds_(thresholds),
sampling_period_ms_(sampling_period_ms),
fast_rampup_(true),
@@ -86,7 +265,6 @@
framedrop_percent_media_opt_(5 * 30),
framedrop_percent_all_(5 * 30),
experiment_enabled_(QualityScalingExperiment::Enabled()),
- observed_enough_frames_(false),
min_frames_needed_(
QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
kMinFramesNeededToScale)),
@@ -94,49 +272,33 @@
.InitialScaleFactor()
.value_or(kSamplePeriodScaleFactor)),
scale_factor_(
- QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()),
- adapt_called_(false),
- adapt_failed_(false) {
+ QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
RTC_DCHECK_RUN_ON(&task_checker_);
if (experiment_enabled_) {
config_ = QualityScalingExperiment::GetConfig();
qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
}
- RTC_DCHECK(observer_ != nullptr);
- check_qp_task_ = RepeatingTaskHandle::DelayedStart(
- TaskQueueBase::Current(), TimeDelta::Millis(GetSamplingPeriodMs()),
- [this]() {
- CheckQp();
- return TimeDelta::Millis(GetSamplingPeriodMs());
- });
+ RTC_DCHECK(handler_ != nullptr);
+ StartNextCheckQpTask();
RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
<< ", high: " << thresholds_.high;
}
QualityScaler::~QualityScaler() {
RTC_DCHECK_RUN_ON(&task_checker_);
- check_qp_task_.Stop();
}
-int64_t QualityScaler::GetSamplingPeriodMs() const {
+void QualityScaler::StartNextCheckQpTask() {
RTC_DCHECK_RUN_ON(&task_checker_);
- if (fast_rampup_) {
- return sampling_period_ms_;
+ RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
+ << "A previous CheckQpTask has not completed yet!";
+ CheckQpTask::Result previous_task_result;
+ if (pending_qp_task_) {
+ previous_task_result = pending_qp_task_->result();
}
- if (experiment_enabled_ && !observed_enough_frames_) {
- // Use half the interval while waiting for enough frames.
- return sampling_period_ms_ / 2;
- }
- if (adapt_failed_) {
- // Check shortly again.
- return sampling_period_ms_ / 8;
- }
- if (scale_factor_ && !adapt_called_) {
- // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
- return sampling_period_ms_ * scale_factor_.value();
- }
- return sampling_period_ms_ * initial_scale_factor_;
+ pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
+ pending_qp_task_->StartDelayedTask();
}
void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
@@ -181,12 +343,10 @@
return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
}
-void QualityScaler::CheckQp() {
+QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
RTC_DCHECK_RUN_ON(&task_checker_);
// Should be set through InitEncode -> Should be set by now.
RTC_DCHECK_GE(thresholds_.low, 0);
- adapt_failed_ = false;
- adapt_called_ = false;
// If we have not observed at least this many frames we can't make a good
// scaling decision.
@@ -194,10 +354,8 @@
? framedrop_percent_all_.Size()
: framedrop_percent_media_opt_.Size();
if (frames < min_frames_needed_) {
- observed_enough_frames_ = false;
- return;
+ return CheckQpResult::kInsufficientSamples;
}
- observed_enough_frames_ = true;
// Check if we should scale down due to high frame drop.
const absl::optional<int> drop_rate =
@@ -206,8 +364,7 @@
: framedrop_percent_media_opt_.GetAverageRoundedDown();
if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
- ReportQpHigh();
- return;
+ return CheckQpResult::kHighQp;
}
// Check if we should scale up or down based on QP.
@@ -221,38 +378,14 @@
RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
<< *avg_qp_low << ").";
if (*avg_qp_high > thresholds_.high) {
- ReportQpHigh();
- return;
+ return CheckQpResult::kHighQp;
}
if (*avg_qp_low <= thresholds_.low) {
// QP has been low. We want to try a higher resolution.
- ReportQpLow();
- return;
+ return CheckQpResult::kLowQp;
}
}
-}
-
-void QualityScaler::ReportQpLow() {
- RTC_DCHECK_RUN_ON(&task_checker_);
- ClearSamples();
- observer_->AdaptUp(VideoAdaptationReason::kQuality);
- adapt_called_ = true;
-}
-
-void QualityScaler::ReportQpHigh() {
- RTC_DCHECK_RUN_ON(&task_checker_);
-
- if (observer_->AdaptDown(VideoAdaptationReason::kQuality)) {
- ClearSamples();
- } else {
- adapt_failed_ = true;
- }
-
- // If we've scaled down, wait longer before scaling up again.
- if (fast_rampup_) {
- fast_rampup_ = false;
- }
- adapt_called_ = true;
+ return CheckQpResult::kNormalQp;
}
void QualityScaler::ClearSamples() {
@@ -265,4 +398,13 @@
if (qp_smoother_low_)
qp_smoother_low_->Reset();
}
+
+QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}
+
+QualityScalerQpUsageHandlerCallbackInterface::
+ QualityScalerQpUsageHandlerCallbackInterface() {}
+
+QualityScalerQpUsageHandlerCallbackInterface::
+ ~QualityScalerQpUsageHandlerCallbackInterface() {}
+
} // namespace webrtc
diff --git a/modules/video_coding/utility/quality_scaler.h b/modules/video_coding/utility/quality_scaler.h
index d6fd599..cfd2fce 100644
--- a/modules/video_coding/utility/quality_scaler.h
+++ b/modules/video_coding/utility/quality_scaler.h
@@ -17,45 +17,30 @@
#include <memory>
#include "absl/types/optional.h"
-#include "api/video/video_adaptation_reason.h"
+#include "api/scoped_refptr.h"
#include "api/video_codecs/video_encoder.h"
#include "rtc_base/experiments/quality_scaling_experiment.h"
#include "rtc_base/numerics/moving_average.h"
+#include "rtc_base/ref_count.h"
+#include "rtc_base/ref_counted_object.h"
#include "rtc_base/synchronization/sequence_checker.h"
#include "rtc_base/task_queue.h"
-#include "rtc_base/task_utils/repeating_task.h"
namespace webrtc {
-// An interface for signaling requests to limit or increase the resolution or
-// framerate of the captured video stream.
-// TODO(hbos): Can we remove AdaptationObserverInterface in favor of
-// ResourceUsageListener? If we need to adapt that is because of resource usage.
-// A multi-stream and multi-resource aware solution needs to sparate the notion
-// of being resource constrained from the decision to downgrade a specific
-// stream.
-class AdaptationObserverInterface {
- public:
- // Called to signal that we can handle larger or more frequent frames.
- virtual void AdaptUp(VideoAdaptationReason reason) = 0;
- // Called to signal that the source should reduce the resolution or framerate.
- // Returns false if a downgrade was requested but the request did not result
- // in a new limiting resolution or fps.
- virtual bool AdaptDown(VideoAdaptationReason reason) = 0;
-
- protected:
- virtual ~AdaptationObserverInterface() {}
-};
+class QualityScalerQpUsageHandlerCallbackInterface;
+class QualityScalerQpUsageHandlerInterface;
// QualityScaler runs asynchronously and monitors QP values of encoded frames.
-// It holds a reference to an AdaptationObserverInterface implementation to
-// signal an intent to scale up or down.
+// It holds a reference to a QualityScalerQpUsageHandlerInterface implementation
+// to signal an overuse or underuse of QP (which indicate a desire to scale the
+// video stream down or up).
class QualityScaler {
public:
- // Construct a QualityScaler with given |thresholds| and |observer|.
+ // Construct a QualityScaler with given |thresholds| and |handler|.
// This starts the quality scaler periodically checking what the average QP
// has been recently.
- QualityScaler(AdaptationObserverInterface* observer,
+ QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds);
virtual ~QualityScaler();
// Should be called each time a frame is dropped at encoding.
@@ -69,21 +54,34 @@
// The following members declared protected for testing purposes.
protected:
- QualityScaler(AdaptationObserverInterface* observer,
+ QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds,
int64_t sampling_period_ms);
private:
class QpSmoother;
+ class CheckQpTask;
+ class CheckQpTaskHandlerCallback;
- void CheckQp();
+ enum class CheckQpResult {
+ kInsufficientSamples,
+ kNormalQp,
+ kHighQp,
+ kLowQp,
+ };
+
+ // Starts checking for QP in a delayed task. When the resulting CheckQpTask
+ // completes, it will invoke this method again, ensuring that we always
+ // periodically check for QP. See CheckQpTask for more details. We never run
+ // more than one CheckQpTask at a time.
+ void StartNextCheckQpTask();
+
+ CheckQpResult CheckQp() const;
void ClearSamples();
- void ReportQpLow();
- void ReportQpHigh();
- int64_t GetSamplingPeriodMs() const;
- RepeatingTaskHandle check_qp_task_ RTC_GUARDED_BY(&task_checker_);
- AdaptationObserverInterface* const observer_ RTC_GUARDED_BY(&task_checker_);
+ std::unique_ptr<CheckQpTask> pending_qp_task_ RTC_GUARDED_BY(&task_checker_);
+ QualityScalerQpUsageHandlerInterface* const handler_
+ RTC_GUARDED_BY(&task_checker_);
SequenceChecker task_checker_;
VideoEncoder::QpThresholds thresholds_ RTC_GUARDED_BY(&task_checker_);
@@ -99,14 +97,55 @@
QualityScalingExperiment::Config config_ RTC_GUARDED_BY(&task_checker_);
std::unique_ptr<QpSmoother> qp_smoother_high_ RTC_GUARDED_BY(&task_checker_);
std::unique_ptr<QpSmoother> qp_smoother_low_ RTC_GUARDED_BY(&task_checker_);
- bool observed_enough_frames_ RTC_GUARDED_BY(&task_checker_);
const size_t min_frames_needed_;
const double initial_scale_factor_;
const absl::optional<double> scale_factor_;
- bool adapt_called_ RTC_GUARDED_BY(&task_checker_);
- bool adapt_failed_ RTC_GUARDED_BY(&task_checker_);
};
+
+// Reacts to QP being too high or too low. For best quality, when QP is high it
+// is desired to decrease the resolution or frame rate of the stream and when QP
+// is low it is desired to increase the resolution or frame rate of the stream.
+// Whether to reconfigure the stream is ultimately up to the handler, which is
+// able to respond asynchronously.
+class QualityScalerQpUsageHandlerInterface {
+ public:
+ virtual ~QualityScalerQpUsageHandlerInterface();
+
+ // Reacts to QP usage being too high or too low. The |callback| MUST be
+ // invoked when the handler is done, allowing the QualityScaler to resume
+ // checking for QP.
+ virtual void OnReportQpUsageHigh(
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
+ callback) = 0;
+ virtual void OnReportQpUsageLow(
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
+ callback) = 0;
+};
+
+// When QP is reported as high or low by the QualityScaler, it pauses checking
+// for QP until the QP usage has been handled. When OnQpUsageHandled() is
+// invoked, the QualityScaler resumes checking for QP. This ensures that if the
+// stream is reconfigured in response to QP usage we do not include QP samples
+// from before the reconfiguration the next time we check for QP.
+//
+// OnQpUsageHandled() MUST be invoked exactly once before this object is
+// destroyed.
+class QualityScalerQpUsageHandlerCallbackInterface
+ : public rtc::RefCountedObject<rtc::RefCountInterface> {
+ public:
+ virtual ~QualityScalerQpUsageHandlerCallbackInterface();
+
+ // If |clear_qp_samples| is true, existing QP samples are cleared before the
+ // next time QualityScaler checks for QP. This is usually a good idea when the
+ // stream is reconfigured. If |clear_qp_samples| is false, samples are not
+ // cleared and QualityScaler increases its frequency of checking for QP.
+ virtual void OnQpUsageHandled(bool clear_qp_samples) = 0;
+
+ protected:
+ QualityScalerQpUsageHandlerCallbackInterface();
+};
+
} // namespace webrtc
#endif // MODULES_VIDEO_CODING_UTILITY_QUALITY_SCALER_H_
diff --git a/modules/video_coding/utility/quality_scaler_unittest.cc b/modules/video_coding/utility/quality_scaler_unittest.cc
index a000504..275b327 100644
--- a/modules/video_coding/utility/quality_scaler_unittest.cc
+++ b/modules/video_coding/utility/quality_scaler_unittest.cc
@@ -13,7 +13,6 @@
#include <memory>
#include <string>
-#include "api/video/video_adaptation_reason.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
#include "rtc_base/task_queue_for_test.h"
@@ -29,31 +28,45 @@
static const size_t kDefaultTimeoutMs = 150;
} // namespace
-class MockAdaptationObserver : public AdaptationObserverInterface {
+class MockQpUsageHandler : public QualityScalerQpUsageHandlerInterface {
public:
- virtual ~MockAdaptationObserver() {}
+ virtual ~MockQpUsageHandler() {}
- void AdaptUp(VideoAdaptationReason r) override {
- adapt_up_events_++;
- event.Set();
- }
- bool AdaptDown(VideoAdaptationReason r) override {
+ // QualityScalerQpUsageHandlerInterface implementation.
+ void OnReportQpUsageHigh(
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback)
+ override {
+ callback_ = callback;
adapt_down_events_++;
event.Set();
- return true;
+ if (synchronously_invoke_callback)
+ callback_->OnQpUsageHandled(true);
+ }
+
+ void OnReportQpUsageLow(
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback)
+ override {
+ callback_ = callback;
+ adapt_up_events_++;
+ event.Set();
+ if (synchronously_invoke_callback)
+ callback_->OnQpUsageHandled(true);
}
rtc::Event event;
int adapt_up_events_ = 0;
int adapt_down_events_ = 0;
+ bool synchronously_invoke_callback = true;
+ rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface> callback_ =
+ nullptr;
};
// Pass a lower sampling period to speed up the tests.
class QualityScalerUnderTest : public QualityScaler {
public:
- explicit QualityScalerUnderTest(AdaptationObserverInterface* observer,
+ explicit QualityScalerUnderTest(QualityScalerQpUsageHandlerInterface* handler,
VideoEncoder::QpThresholds thresholds)
- : QualityScaler(observer, thresholds, 5) {}
+ : QualityScaler(handler, thresholds, 5) {}
};
class QualityScalerTest : public ::testing::Test,
@@ -70,11 +83,11 @@
QualityScalerTest()
: scoped_field_trial_(GetParam()),
task_queue_("QualityScalerTestQueue"),
- observer_(new MockAdaptationObserver()) {
+ handler_(new MockQpUsageHandler()) {
task_queue_.SendTask(
[this] {
qs_ = std::unique_ptr<QualityScaler>(new QualityScalerUnderTest(
- observer_.get(), VideoEncoder::QpThresholds(kLowQp, kHighQp)));
+ handler_.get(), VideoEncoder::QpThresholds(kLowQp, kHighQp)));
},
RTC_FROM_HERE);
}
@@ -108,7 +121,7 @@
test::ScopedFieldTrials scoped_field_trial_;
TaskQueueForTest task_queue_;
std::unique_ptr<QualityScaler> qs_;
- std::unique_ptr<MockAdaptationObserver> observer_;
+ std::unique_ptr<MockQpUsageHandler> handler_;
};
INSTANTIATE_TEST_SUITE_P(
@@ -120,25 +133,25 @@
TEST_P(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
task_queue_.SendTask([this] { TriggerScale(kScaleDown); }, RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, KeepsScaleAtHighQp) {
task_queue_.SendTask([this] { TriggerScale(kKeepScaleAtHighQp); },
RTC_FROM_HERE);
- EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(0, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(0, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, DownscalesAboveHighQp) {
task_queue_.SendTask([this] { TriggerScale(kScaleDownAboveHighQp); },
RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
@@ -151,9 +164,9 @@
}
},
RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
@@ -165,9 +178,9 @@
}
},
RTC_FROM_HERE);
- EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(0, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(0, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, DownscalesAfterTwoThirdsIfFieldTrialEnabled) {
@@ -181,35 +194,35 @@
}
},
RTC_FROM_HERE);
- EXPECT_EQ(kDownScaleExpected, observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(kDownScaleExpected ? 1 : 0, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_EQ(kDownScaleExpected, handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(kDownScaleExpected ? 1 : 0, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, KeepsScaleOnNormalQp) {
task_queue_.SendTask([this] { TriggerScale(kKeepScaleAboveLowQp); },
RTC_FROM_HERE);
- EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(0, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(0, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, UpscalesAfterLowQp) {
task_queue_.SendTask([this] { TriggerScale(kScaleUp); }, RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(0, observer_->adapt_down_events_);
- EXPECT_EQ(1, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(0, handler_->adapt_down_events_);
+ EXPECT_EQ(1, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, ScalesDownAndBackUp) {
task_queue_.SendTask([this] { TriggerScale(kScaleDown); }, RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
task_queue_.SendTask([this] { TriggerScale(kScaleUp); }, RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(1, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(1, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, DoesNotScaleUntilEnoughFramesObserved) {
@@ -221,7 +234,7 @@
}
},
RTC_FROM_HERE);
- EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs));
task_queue_.SendTask(
[this] {
// Send 1 more. Enough frames observed, should result in an adapt
@@ -229,9 +242,9 @@
qs_->ReportQp(kLowQp, 0);
},
RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(0, observer_->adapt_down_events_);
- EXPECT_EQ(1, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(0, handler_->adapt_down_events_);
+ EXPECT_EQ(1, handler_->adapt_up_events_);
// Samples should be cleared after an adapt request.
task_queue_.SendTask(
@@ -240,9 +253,9 @@
qs_->ReportQp(kLowQp, 0);
},
RTC_FROM_HERE);
- EXPECT_FALSE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(0, observer_->adapt_down_events_);
- EXPECT_EQ(1, observer_->adapt_up_events_);
+ EXPECT_FALSE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(0, handler_->adapt_down_events_);
+ EXPECT_EQ(1, handler_->adapt_up_events_);
}
TEST_P(QualityScalerTest, ScalesDownAndBackUpWithMinFramesNeeded) {
@@ -253,9 +266,9 @@
}
},
RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(0, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(0, handler_->adapt_up_events_);
// Samples cleared.
task_queue_.SendTask(
[this] {
@@ -264,9 +277,39 @@
}
},
RTC_FROM_HERE);
- EXPECT_TRUE(observer_->event.Wait(kDefaultTimeoutMs));
- EXPECT_EQ(1, observer_->adapt_down_events_);
- EXPECT_EQ(1, observer_->adapt_up_events_);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ EXPECT_EQ(1, handler_->adapt_up_events_);
+}
+
+TEST_P(QualityScalerTest, CheckingQpAgainRequiresResolvingCallback) {
+ handler_->synchronously_invoke_callback = false;
+ task_queue_.SendTask([this] { TriggerScale(kScaleDown); }, RTC_FROM_HERE);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ // Without invoking the callback, another downscale should not happen.
+ handler_->event.Reset();
+ rtc::Event event;
+ task_queue_.SendTask(
+ [this, &event] {
+ TriggerScale(kScaleDown);
+ event.Set();
+ },
+ RTC_FROM_HERE);
+ EXPECT_TRUE(event.Wait(kDefaultTimeoutMs));
+ EXPECT_FALSE(handler_->event.Wait(0));
+ EXPECT_EQ(1, handler_->adapt_down_events_);
+ // Resume checking for QP again by invoking the callback.
+ task_queue_.SendTask(
+ [this] {
+ handler_->callback_->OnQpUsageHandled(true);
+ TriggerScale(kScaleDown);
+ },
+ RTC_FROM_HERE);
+ EXPECT_TRUE(handler_->event.Wait(kDefaultTimeoutMs));
+ EXPECT_EQ(2, handler_->adapt_down_events_);
+ task_queue_.SendTask([this] { handler_->callback_->OnQpUsageHandled(true); },
+ RTC_FROM_HERE);
}
} // namespace webrtc