modules/video_coding/utility/quality_scaler.cc

/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/video_coding/utility/quality_scaler.h"

#include <memory>
#include <utility>

#include "api/video/video_adaptation_reason.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/task_utils/to_queued_task.h"
#include "rtc_base/weak_ptr.h"

// TODO(kthelgason): Some versions of Android have issues with log2.
// See https://code.google.com/p/android/issues/detail?id=212634 for details
#if defined(WEBRTC_ANDROID)
#define log2(x) (log(x) / log(2))
#endif

namespace webrtc {

namespace {
// TODO(nisse): Delete, delegate to encoders.
// Threshold constant used until first downscale (to permit fast rampup).
static const int kMeasureMs = 2000;
static const float kSamplePeriodScaleFactor = 2.5;
static const int kFramedropPercentThreshold = 60;
static const size_t kMinFramesNeededToScale = 2 * 30;

}  // namespace

class QualityScaler::QpSmoother {
 public:
  explicit QpSmoother(float alpha)
      : alpha_(alpha),
        // The initial value of last_sample_ms doesn't matter since the smoother
        // will ignore the time delta for the first update.
        last_sample_ms_(0),
        smoother_(alpha) {}

  absl::optional<int> GetAvg() const {
    float value = smoother_.filtered();
    if (value == rtc::ExpFilter::kValueUndefined) {
      return absl::nullopt;
    }
    return static_cast<int>(value);
  }

  void Add(float sample, int64_t time_sent_us) {
    int64_t now_ms = time_sent_us / 1000;
    smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
    last_sample_ms_ = now_ms;
  }

  void Reset() { smoother_.Reset(alpha_); }

 private:
  const float alpha_;
  int64_t last_sample_ms_;
  rtc::ExpFilter smoother_;
};

// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
// task will either run to completion and trigger a new task being queued, or it
// will be destroyed because the QualityScaler is destroyed.
//
// When high or low QP is reported, the task will be pending until a callback is
// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
// asynchronously and prevents checking for QP until the stream has potentially
// been reconfigured.
class QualityScaler::CheckQpTask {
 public:
  // The result of one CheckQpTask may influence the delay of the next
  // CheckQpTask.
  struct Result {
    bool observed_enough_frames = false;
    bool qp_usage_reported = false;
    bool clear_qp_samples = false;
  };

  CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
      : quality_scaler_(quality_scaler),
        state_(State::kNotStarted),
        previous_task_result_(previous_task_result),
        weak_ptr_factory_(this) {}

  void StartDelayedTask() {
    RTC_DCHECK_EQ(state_, State::kNotStarted);
    state_ = State::kCheckingQp;
    TaskQueueBase::Current()->PostDelayedTask(
        ToQueuedTask([this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
          if (!this_weak_ptr) {
            // The task has been cancelled through destruction.
            return;
          }
          RTC_DCHECK_EQ(state_, State::kCheckingQp);
          RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
          switch (quality_scaler_->CheckQp()) {
            case QualityScaler::CheckQpResult::kInsufficientSamples: {
              result_.observed_enough_frames = false;
              // After this line, |this| may be deleted.
              DoCompleteTask();
              return;
            }
            case QualityScaler::CheckQpResult::kNormalQp: {
              result_.observed_enough_frames = true;
              // After this line, |this| may be deleted.
              DoCompleteTask();
              return;
            }
            case QualityScaler::CheckQpResult::kHighQp: {
              result_.observed_enough_frames = true;
              result_.qp_usage_reported = true;
              state_ = State::kAwaitingQpUsageHandled;
              rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
                  callback = ConstructCallback();
              quality_scaler_->fast_rampup_ = false;
              // After this line, |this| may be deleted.
              quality_scaler_->handler_->OnReportQpUsageHigh(callback);
              return;
            }
            case QualityScaler::CheckQpResult::kLowQp: {
              result_.observed_enough_frames = true;
              result_.qp_usage_reported = true;
              state_ = State::kAwaitingQpUsageHandled;
              rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>
                  callback = ConstructCallback();
              // After this line, |this| may be deleted.
              quality_scaler_->handler_->OnReportQpUsageLow(callback);
              return;
            }
          }
        }),
        GetCheckingQpDelayMs());
  }

  void OnQpUsageHandled(bool clear_qp_samples) {
    RTC_DCHECK_EQ(state_, State::kAwaitingQpUsageHandled);
    result_.clear_qp_samples = clear_qp_samples;
    if (clear_qp_samples)
      quality_scaler_->ClearSamples();
    DoCompleteTask();
  }

  bool HasCompletedTask() const { return state_ == State::kCompleted; }

  Result result() const {
    RTC_DCHECK(HasCompletedTask());
    return result_;
  }

 private:
  enum class State {
    kNotStarted,
    kCheckingQp,
    kAwaitingQpUsageHandled,
    kCompleted,
  };

  // Defined after the definition of QualityScaler::CheckQpTaskHandlerCallback.
  // Gets around a forward declaration issue.
  rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
  ConstructCallback();

  // Determines the sampling period of CheckQpTasks.
  int64_t GetCheckingQpDelayMs() const {
    RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
    if (quality_scaler_->fast_rampup_) {
      return quality_scaler_->sampling_period_ms_;
    }
    if (quality_scaler_->experiment_enabled_ &&
        !previous_task_result_.observed_enough_frames) {
      // Use half the interval while waiting for enough frames.
      return quality_scaler_->sampling_period_ms_ / 2;
    }
    if (!previous_task_result_.clear_qp_samples) {
      // Check shortly again.
      return quality_scaler_->sampling_period_ms_ / 8;
    }
    if (quality_scaler_->scale_factor_ &&
        !previous_task_result_.qp_usage_reported) {
      // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
      return quality_scaler_->sampling_period_ms_ *
             quality_scaler_->scale_factor_.value();
    }
    return quality_scaler_->sampling_period_ms_ *
           quality_scaler_->initial_scale_factor_;
  }

  void DoCompleteTask() {
    RTC_DCHECK(state_ == State::kCheckingQp ||
               state_ == State::kAwaitingQpUsageHandled);
    state_ = State::kCompleted;
    // Starting the next task deletes the pending task. After this line, |this|
    // has been deleted.
    quality_scaler_->StartNextCheckQpTask();
  }

  QualityScaler* const quality_scaler_;
  State state_;
  const Result previous_task_result_;
  Result result_;

  rtc::WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
};

class QualityScaler::CheckQpTaskHandlerCallback
    : public QualityScalerQpUsageHandlerCallbackInterface {
 public:
  CheckQpTaskHandlerCallback(
      rtc::WeakPtr<QualityScaler::CheckQpTask> check_qp_task)
      : QualityScalerQpUsageHandlerCallbackInterface(),
        check_qp_task_(std::move(check_qp_task)),
        was_handled_(false) {}

  ~CheckQpTaskHandlerCallback() { RTC_DCHECK(was_handled_); }

  void OnQpUsageHandled(bool clear_qp_samples) {
    RTC_DCHECK(!was_handled_);
    was_handled_ = true;
    if (!check_qp_task_) {
      // The task has been cancelled through destruction; the result of the
      // operation is ignored.
      return;
    }
    check_qp_task_->OnQpUsageHandled(clear_qp_samples);
  }

 private:
  // The callback may outlive the QualityScaler and its task.
  rtc::WeakPtr<QualityScaler::CheckQpTask> const check_qp_task_;
  bool was_handled_;
};

rtc::scoped_refptr<QualityScaler::CheckQpTaskHandlerCallback>
QualityScaler::CheckQpTask::ConstructCallback() {
  return new CheckQpTaskHandlerCallback(weak_ptr_factory_.GetWeakPtr());
}

QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
                             VideoEncoder::QpThresholds thresholds)
    : QualityScaler(handler, thresholds, kMeasureMs) {}

// Protected ctor, should not be called directly.
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
                             VideoEncoder::QpThresholds thresholds,
                             int64_t sampling_period_ms)
    : handler_(handler),
      thresholds_(thresholds),
      sampling_period_ms_(sampling_period_ms),
      fast_rampup_(true),
      // Arbitrarily choose size based on 30 fps for 5 seconds.
      average_qp_(5 * 30),
      framedrop_percent_media_opt_(5 * 30),
      framedrop_percent_all_(5 * 30),
      experiment_enabled_(QualityScalingExperiment::Enabled()),
      min_frames_needed_(
          QualityScalerSettings::ParseFromFieldTrials().MinFrames().value_or(
              kMinFramesNeededToScale)),
      initial_scale_factor_(QualityScalerSettings::ParseFromFieldTrials()
                                .InitialScaleFactor()
                                .value_or(kSamplePeriodScaleFactor)),
      scale_factor_(
          QualityScalerSettings::ParseFromFieldTrials().ScaleFactor()) {
  RTC_DCHECK_RUN_ON(&task_checker_);
  if (experiment_enabled_) {
    config_ = QualityScalingExperiment::GetConfig();
    qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
    qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
  }
  RTC_DCHECK(handler_ != nullptr);
  StartNextCheckQpTask();
  RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
                   << ", high: " << thresholds_.high;
}

QualityScaler::~QualityScaler() {
  RTC_DCHECK_RUN_ON(&task_checker_);
}

void QualityScaler::StartNextCheckQpTask() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
      << "A previous CheckQpTask has not completed yet!";
  CheckQpTask::Result previous_task_result;
  if (pending_qp_task_) {
    previous_task_result = pending_qp_task_->result();
  }
  pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
  pending_qp_task_->StartDelayedTask();
}

void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
  RTC_DCHECK_RUN_ON(&task_checker_);
  thresholds_ = thresholds;
}

void QualityScaler::ReportDroppedFrameByMediaOpt() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_media_opt_.AddSample(100);
  framedrop_percent_all_.AddSample(100);
}

void QualityScaler::ReportDroppedFrameByEncoder() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_all_.AddSample(100);
}

void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_media_opt_.AddSample(0);
  framedrop_percent_all_.AddSample(0);
  average_qp_.AddSample(qp);
  if (qp_smoother_high_)
    qp_smoother_high_->Add(qp, time_sent_us);
  if (qp_smoother_low_)
    qp_smoother_low_->Add(qp, time_sent_us);
}

bool QualityScaler::QpFastFilterLow() const {
  RTC_DCHECK_RUN_ON(&task_checker_);
  size_t num_frames = config_.use_all_drop_reasons
                          ? framedrop_percent_all_.Size()
                          : framedrop_percent_media_opt_.Size();
  const size_t kMinNumFrames = 10;
  if (num_frames < kMinNumFrames) {
    return false;  // Wait for more frames before making a decision.
  }
  absl::optional<int> avg_qp_high = qp_smoother_high_
                                        ? qp_smoother_high_->GetAvg()
                                        : average_qp_.GetAverageRoundedDown();
  return (avg_qp_high) ? (avg_qp_high.value() <= thresholds_.low) : false;
}

QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
  RTC_DCHECK_RUN_ON(&task_checker_);
  // Should be set through InitEncode -> Should be set by now.
  RTC_DCHECK_GE(thresholds_.low, 0);

  // If we have not observed at least this many frames we can't make a good
  // scaling decision.
  const size_t frames = config_.use_all_drop_reasons
                            ? framedrop_percent_all_.Size()
                            : framedrop_percent_media_opt_.Size();
  if (frames < min_frames_needed_) {
    return CheckQpResult::kInsufficientSamples;
  }

  // Check if we should scale down due to high frame drop.
  const absl::optional<int> drop_rate =
      config_.use_all_drop_reasons
          ? framedrop_percent_all_.GetAverageRoundedDown()
          : framedrop_percent_media_opt_.GetAverageRoundedDown();
  if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
    RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
    return CheckQpResult::kHighQp;
  }

  // Check if we should scale up or down based on QP.
  const absl::optional<int> avg_qp_high =
      qp_smoother_high_ ? qp_smoother_high_->GetAvg()
                        : average_qp_.GetAverageRoundedDown();
  const absl::optional<int> avg_qp_low =
      qp_smoother_low_ ? qp_smoother_low_->GetAvg()
                       : average_qp_.GetAverageRoundedDown();
  if (avg_qp_high && avg_qp_low) {
    RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
                     << *avg_qp_low << ").";
    if (*avg_qp_high > thresholds_.high) {
      return CheckQpResult::kHighQp;
    }
    if (*avg_qp_low <= thresholds_.low) {
      // QP has been low. We want to try a higher resolution.
      return CheckQpResult::kLowQp;
    }
  }
  return CheckQpResult::kNormalQp;
}

void QualityScaler::ClearSamples() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_media_opt_.Reset();
  framedrop_percent_all_.Reset();
  average_qp_.Reset();
  if (qp_smoother_high_)
    qp_smoother_high_->Reset();
  if (qp_smoother_low_)
    qp_smoother_low_->Reset();
}

QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}

QualityScalerQpUsageHandlerCallbackInterface::
    QualityScalerQpUsageHandlerCallbackInterface() {}

QualityScalerQpUsageHandlerCallbackInterface::
    ~QualityScalerQpUsageHandlerCallbackInterface() {}

}  // namespace webrtc