webrtc/modules/video_coding/media_optimization.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/video_coding/media_optimization.h"

#include "webrtc/base/logging.h"
#include "webrtc/modules/video_coding/content_metrics_processing.h"
#include "webrtc/modules/video_coding/qm_select.h"
#include "webrtc/modules/video_coding/utility/frame_dropper.h"
#include "webrtc/system_wrappers/include/clock.h"

namespace webrtc {
namespace media_optimization {
namespace {
void UpdateProtectionCallback(
    VCMProtectionMethod* selected_method,
    uint32_t* video_rate_bps,
    uint32_t* nack_overhead_rate_bps,
    uint32_t* fec_overhead_rate_bps,
    VCMProtectionCallback* video_protection_callback) {
  FecProtectionParams delta_fec_params;
  FecProtectionParams key_fec_params;
  // Get the FEC code rate for Key frames (set to 0 when NA).
  key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK();

  // Get the FEC code rate for Delta frames (set to 0 when NA).
  delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD();

  // The RTP module currently requires the same |max_fec_frames| for both
  // key and delta frames.
  delta_fec_params.max_fec_frames = selected_method->MaxFramesFec();
  key_fec_params.max_fec_frames = selected_method->MaxFramesFec();

  // Set the FEC packet mask type. |kFecMaskBursty| is more effective for
  // consecutive losses and little/no packet re-ordering. As we currently
  // do not have feedback data on the degree of correlated losses and packet
  // re-ordering, we keep default setting to |kFecMaskRandom| for now.
  delta_fec_params.fec_mask_type = kFecMaskRandom;
  key_fec_params.fec_mask_type = kFecMaskRandom;

  // TODO(Marco): Pass FEC protection values per layer.
  video_protection_callback->ProtectionRequest(
      &delta_fec_params, &key_fec_params, video_rate_bps,
      nack_overhead_rate_bps, fec_overhead_rate_bps);
}
}  // namespace

struct MediaOptimization::EncodedFrameSample {
  EncodedFrameSample(size_t size_bytes,
                     uint32_t timestamp,
                     int64_t time_complete_ms)
      : size_bytes(size_bytes),
        timestamp(timestamp),
        time_complete_ms(time_complete_ms) {}

  size_t size_bytes;
  uint32_t timestamp;
  int64_t time_complete_ms;
};

MediaOptimization::MediaOptimization(Clock* clock)
    : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
      clock_(clock),
      max_bit_rate_(0),
      send_codec_type_(kVideoCodecUnknown),
      codec_width_(0),
      codec_height_(0),
      user_frame_rate_(0),
      frame_dropper_(new FrameDropper),
      loss_prot_logic_(
          new VCMLossProtectionLogic(clock_->TimeInMilliseconds())),
      fraction_lost_(0),
      send_statistics_zero_encode_(0),
      max_payload_size_(1460),
      video_target_bitrate_(0),
      incoming_frame_rate_(0),
      enable_qm_(false),
      encoded_frame_samples_(),
      avg_sent_bit_rate_bps_(0),
      avg_sent_framerate_(0),
      key_frame_cnt_(0),
      delta_frame_cnt_(0),
      content_(new VCMContentMetricsProcessing()),
      qm_resolution_(new VCMQmResolution()),
      last_qm_update_time_(0),
      last_change_time_(0),
      num_layers_(0),
      suspension_enabled_(false),
      video_suspended_(false),
      suspension_threshold_bps_(0),
      suspension_window_bps_(0) {
  memset(send_statistics_, 0, sizeof(send_statistics_));
  memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
}

MediaOptimization::~MediaOptimization(void) {
  loss_prot_logic_->Release();
}

void MediaOptimization::Reset() {
  CriticalSectionScoped lock(crit_sect_.get());
  SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0,
                          max_payload_size_);
  memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
  incoming_frame_rate_ = 0.0;
  frame_dropper_->Reset();
  loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
  frame_dropper_->SetRates(0, 0);
  content_->Reset();
  qm_resolution_->Reset();
  loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_);
  loss_prot_logic_->Reset(clock_->TimeInMilliseconds());
  send_statistics_zero_encode_ = 0;
  video_target_bitrate_ = 0;
  codec_width_ = 0;
  codec_height_ = 0;
  user_frame_rate_ = 0;
  key_frame_cnt_ = 0;
  delta_frame_cnt_ = 0;
  last_qm_update_time_ = 0;
  last_change_time_ = 0;
  encoded_frame_samples_.clear();
  avg_sent_bit_rate_bps_ = 0;
  num_layers_ = 1;
}

void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type,
                                        int32_t max_bit_rate,
                                        uint32_t target_bitrate,
                                        uint16_t width,
                                        uint16_t height,
                                        uint32_t frame_rate,
                                        int num_layers,
                                        int32_t mtu) {
  CriticalSectionScoped lock(crit_sect_.get());
  SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate,
                          target_bitrate, width, height, num_layers, mtu);
}

void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type,
                                                int32_t max_bit_rate,
                                                uint32_t frame_rate,
                                                uint32_t target_bitrate,
                                                uint16_t width,
                                                uint16_t height,
                                                int num_layers,
                                                int32_t mtu) {
  // Everything codec specific should be reset here since this means the codec
  // has changed. If native dimension values have changed, then either user
  // initiated change, or QM initiated change. Will be able to determine only
  // after the processing of the first frame.
  last_change_time_ = clock_->TimeInMilliseconds();
  content_->Reset();
  content_->UpdateFrameRate(frame_rate);

  max_bit_rate_ = max_bit_rate;
  send_codec_type_ = send_codec_type;
  video_target_bitrate_ = target_bitrate;
  float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
  loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
  loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate));
  loss_prot_logic_->UpdateFrameSize(width, height);
  loss_prot_logic_->UpdateNumLayers(num_layers);
  frame_dropper_->Reset();
  frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
  user_frame_rate_ = static_cast<float>(frame_rate);
  codec_width_ = width;
  codec_height_ = height;
  num_layers_ = (num_layers <= 1) ? 1 : num_layers;  // Can also be zero.
  max_payload_size_ = mtu;
  qm_resolution_->Initialize(target_bitrate_kbps, user_frame_rate_,
                             codec_width_, codec_height_, num_layers_);
}

uint32_t MediaOptimization::SetTargetRates(
    uint32_t target_bitrate,
    uint8_t fraction_lost,
    int64_t round_trip_time_ms,
    VCMProtectionCallback* protection_callback,
    VCMQMSettingsCallback* qmsettings_callback) {
  CriticalSectionScoped lock(crit_sect_.get());
  VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod();
  float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
  loss_prot_logic_->UpdateBitRate(target_bitrate_kbps);
  loss_prot_logic_->UpdateRtt(round_trip_time_ms);

  // Get frame rate for encoder: this is the actual/sent frame rate.
  float actual_frame_rate = SentFrameRateInternal();

  // Sanity check.
  if (actual_frame_rate < 1.0) {
    actual_frame_rate = 1.0;
  }

  // Update frame rate for the loss protection logic class: frame rate should
  // be the actual/sent rate.
  loss_prot_logic_->UpdateFrameRate(actual_frame_rate);

  fraction_lost_ = fraction_lost;

  // Returns the filtered packet loss, used for the protection setting.
  // The filtered loss may be the received loss (no filter), or some
  // filtered value (average or max window filter).
  // Use max window filter for now.
  FilterPacketLossMode filter_mode = kMaxFilter;
  uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss(
      clock_->TimeInMilliseconds(), filter_mode, fraction_lost);

  // For now use the filtered loss for computing the robustness settings.
  loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc);

  // Rate cost of the protection methods.
  float protection_overhead_rate = 0.0f;

  // Update protection settings, when applicable.
  float sent_video_rate_kbps = 0.0f;
  if (loss_prot_logic_->SelectedType() != kNone) {
    // Update method will compute the robustness settings for the given
    // protection method and the overhead cost
    // the protection method is set by the user via SetVideoProtection.
    loss_prot_logic_->UpdateMethod();

    // Update protection callback with protection settings.
    uint32_t sent_video_rate_bps = 0;
    uint32_t sent_nack_rate_bps = 0;
    uint32_t sent_fec_rate_bps = 0;
    // Get the bit cost of protection method, based on the amount of
    // overhead data actually transmitted (including headers) the last
    // second.
    if (protection_callback) {
      UpdateProtectionCallback(selected_method, &sent_video_rate_bps,
                               &sent_nack_rate_bps, &sent_fec_rate_bps,
                               protection_callback);
    }
    uint32_t sent_total_rate_bps =
        sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps;
    // Estimate the overhead costs of the next second as staying the same
    // wrt the source bitrate.
    if (sent_total_rate_bps > 0) {
      protection_overhead_rate =
          static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) /
          sent_total_rate_bps;
    }
    // Cap the overhead estimate to 50%.
    if (protection_overhead_rate > 0.5)
      protection_overhead_rate = 0.5;

    // Get the effective packet loss for encoder ER when applicable. Should be
    // passed to encoder via fraction_lost.
    packet_loss_enc = selected_method->RequiredPacketLossER();
    sent_video_rate_kbps = static_cast<float>(sent_video_rate_bps) / 1000.0f;
  }

  // Source coding rate: total rate - protection overhead.
  video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate);

  // Cap target video bitrate to codec maximum.
  if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) {
    video_target_bitrate_ = max_bit_rate_;
  }

  // Update encoding rates following protection settings.
  float target_video_bitrate_kbps =
      static_cast<float>(video_target_bitrate_) / 1000.0f;
  frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_);

  if (enable_qm_ && qmsettings_callback) {
    // Update QM with rates.
    qm_resolution_->UpdateRates(target_video_bitrate_kbps, sent_video_rate_kbps,
                                incoming_frame_rate_, fraction_lost_);
    // Check for QM selection.
    bool select_qm = CheckStatusForQMchange();
    if (select_qm) {
      SelectQuality(qmsettings_callback);
    }
    // Reset the short-term averaged content data.
    content_->ResetShortTermAvgData();
  }

  CheckSuspendConditions();

  return video_target_bitrate_;
}

void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) {
  CriticalSectionScoped lock(crit_sect_.get());
  loss_prot_logic_->SetMethod(method);
}

uint32_t MediaOptimization::InputFrameRate() {
  CriticalSectionScoped lock(crit_sect_.get());
  return InputFrameRateInternal();
}

uint32_t MediaOptimization::InputFrameRateInternal() {
  ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
  return uint32_t(incoming_frame_rate_ + 0.5f);
}

uint32_t MediaOptimization::SentFrameRate() {
  CriticalSectionScoped lock(crit_sect_.get());
  return SentFrameRateInternal();
}

uint32_t MediaOptimization::SentFrameRateInternal() {
  PurgeOldFrameSamples(clock_->TimeInMilliseconds());
  UpdateSentFramerate();
  return avg_sent_framerate_;
}

uint32_t MediaOptimization::SentBitRate() {
  CriticalSectionScoped lock(crit_sect_.get());
  const int64_t now_ms = clock_->TimeInMilliseconds();
  PurgeOldFrameSamples(now_ms);
  UpdateSentBitrate(now_ms);
  return avg_sent_bit_rate_bps_;
}

int32_t MediaOptimization::UpdateWithEncodedData(
    const EncodedImage& encoded_image) {
  size_t encoded_length = encoded_image._length;
  uint32_t timestamp = encoded_image._timeStamp;
  CriticalSectionScoped lock(crit_sect_.get());
  const int64_t now_ms = clock_->TimeInMilliseconds();
  PurgeOldFrameSamples(now_ms);
  if (encoded_frame_samples_.size() > 0 &&
      encoded_frame_samples_.back().timestamp == timestamp) {
    // Frames having the same timestamp are generated from the same input
    // frame. We don't want to double count them, but only increment the
    // size_bytes.
    encoded_frame_samples_.back().size_bytes += encoded_length;
    encoded_frame_samples_.back().time_complete_ms = now_ms;
  } else {
    encoded_frame_samples_.push_back(
        EncodedFrameSample(encoded_length, timestamp, now_ms));
  }
  UpdateSentBitrate(now_ms);
  UpdateSentFramerate();
  if (encoded_length > 0) {
    const bool delta_frame = encoded_image._frameType != kVideoFrameKey;

    frame_dropper_->Fill(encoded_length, delta_frame);
    if (max_payload_size_ > 0 && encoded_length > 0) {
      const float min_packets_per_frame =
          encoded_length / static_cast<float>(max_payload_size_);
      if (delta_frame) {
        loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame,
                                                clock_->TimeInMilliseconds());
      } else {
        loss_prot_logic_->UpdatePacketsPerFrameKey(
            min_packets_per_frame, clock_->TimeInMilliseconds());
      }

      if (enable_qm_) {
        // Update quality select with encoded length.
        qm_resolution_->UpdateEncodedSize(encoded_length);
      }
    }
    if (!delta_frame && encoded_length > 0) {
      loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length));
    }

    // Updating counters.
    if (delta_frame) {
      delta_frame_cnt_++;
    } else {
      key_frame_cnt_++;
    }
  }

  return VCM_OK;
}

void MediaOptimization::EnableQM(bool enable) {
  CriticalSectionScoped lock(crit_sect_.get());
  enable_qm_ = enable;
}

void MediaOptimization::EnableFrameDropper(bool enable) {
  CriticalSectionScoped lock(crit_sect_.get());
  frame_dropper_->Enable(enable);
}

void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps,
                                               int window_bps) {
  CriticalSectionScoped lock(crit_sect_.get());
  assert(threshold_bps > 0 && window_bps >= 0);
  suspension_threshold_bps_ = threshold_bps;
  suspension_window_bps_ = window_bps;
  suspension_enabled_ = true;
  video_suspended_ = false;
}

bool MediaOptimization::IsVideoSuspended() const {
  CriticalSectionScoped lock(crit_sect_.get());
  return video_suspended_;
}

bool MediaOptimization::DropFrame() {
  CriticalSectionScoped lock(crit_sect_.get());
  UpdateIncomingFrameRate();
  // Leak appropriate number of bytes.
  frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
  if (video_suspended_) {
    return true;  // Drop all frames when muted.
  }
  return frame_dropper_->DropFrame();
}

void MediaOptimization::UpdateContentData(
    const VideoContentMetrics* content_metrics) {
  CriticalSectionScoped lock(crit_sect_.get());
  // Updating content metrics.
  if (content_metrics == NULL) {
    // Disable QM if metrics are NULL.
    enable_qm_ = false;
    qm_resolution_->Reset();
  } else {
    content_->UpdateContentData(content_metrics);
  }
}

void MediaOptimization::UpdateIncomingFrameRate() {
  int64_t now = clock_->TimeInMilliseconds();
  if (incoming_frame_times_[0] == 0) {
    // No shifting if this is the first time.
  } else {
    // Shift all times one step.
    for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
      incoming_frame_times_[i + 1] = incoming_frame_times_[i];
    }
  }
  incoming_frame_times_[0] = now;
  ProcessIncomingFrameRate(now);
}

int32_t MediaOptimization::SelectQuality(
    VCMQMSettingsCallback* video_qmsettings_callback) {
  // Reset quantities for QM select.
  qm_resolution_->ResetQM();

  // Update QM will long-term averaged content metrics.
  qm_resolution_->UpdateContent(content_->LongTermAvgData());

  // Select quality mode.
  VCMResolutionScale* qm = NULL;
  int32_t ret = qm_resolution_->SelectResolution(&qm);
  if (ret < 0) {
    return ret;
  }

  // Check for updates to spatial/temporal modes.
  QMUpdate(qm, video_qmsettings_callback);

  // Reset all the rate and related frame counters quantities.
  qm_resolution_->ResetRates();

  // Reset counters.
  last_qm_update_time_ = clock_->TimeInMilliseconds();

  // Reset content metrics.
  content_->Reset();

  return VCM_OK;
}

void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) {
  while (!encoded_frame_samples_.empty()) {
    if (now_ms - encoded_frame_samples_.front().time_complete_ms >
        kBitrateAverageWinMs) {
      encoded_frame_samples_.pop_front();
    } else {
      break;
    }
  }
}

void MediaOptimization::UpdateSentBitrate(int64_t now_ms) {
  if (encoded_frame_samples_.empty()) {
    avg_sent_bit_rate_bps_ = 0;
    return;
  }
  size_t framesize_sum = 0;
  for (FrameSampleList::iterator it = encoded_frame_samples_.begin();
       it != encoded_frame_samples_.end(); ++it) {
    framesize_sum += it->size_bytes;
  }
  float denom = static_cast<float>(
      now_ms - encoded_frame_samples_.front().time_complete_ms);
  if (denom >= 1.0f) {
    avg_sent_bit_rate_bps_ =
        static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f);
  } else {
    avg_sent_bit_rate_bps_ = framesize_sum * 8;
  }
}

void MediaOptimization::UpdateSentFramerate() {
  if (encoded_frame_samples_.size() <= 1) {
    avg_sent_framerate_ = encoded_frame_samples_.size();
    return;
  }
  int denom = encoded_frame_samples_.back().timestamp -
              encoded_frame_samples_.front().timestamp;
  if (denom > 0) {
    avg_sent_framerate_ =
        (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
  } else {
    avg_sent_framerate_ = encoded_frame_samples_.size();
  }
}

bool MediaOptimization::QMUpdate(
    VCMResolutionScale* qm,
    VCMQMSettingsCallback* video_qmsettings_callback) {
  // Check for no change.
  if (!qm->change_resolution_spatial && !qm->change_resolution_temporal) {
    return false;
  }

  // Check for change in frame rate.
  if (qm->change_resolution_temporal) {
    incoming_frame_rate_ = qm->frame_rate;
    // Reset frame rate estimate.
    memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
  }

  // Check for change in frame size.
  if (qm->change_resolution_spatial) {
    codec_width_ = qm->codec_width;
    codec_height_ = qm->codec_height;
  }

  LOG(LS_INFO) << "Media optimizer requests the video resolution to be changed "
                  "to "
               << qm->codec_width << "x" << qm->codec_height << "@"
               << qm->frame_rate;

  // Update VPM with new target frame rate and frame size.
  // Note: use |qm->frame_rate| instead of |_incoming_frame_rate| for updating
  // target frame rate in VPM frame dropper. The quantity |_incoming_frame_rate|
  // will vary/fluctuate, and since we don't want to change the state of the
  // VPM frame dropper, unless a temporal action was selected, we use the
  // quantity |qm->frame_rate| for updating.
  video_qmsettings_callback->SetVideoQMSettings(qm->frame_rate, codec_width_,
                                                codec_height_);
  content_->UpdateFrameRate(qm->frame_rate);
  qm_resolution_->UpdateCodecParameters(qm->frame_rate, codec_width_,
                                        codec_height_);
  return true;
}

// Check timing constraints and look for significant change in:
// (1) scene content,
// (2) target bit rate.
bool MediaOptimization::CheckStatusForQMchange() {
  bool status = true;

  // Check that we do not call QMSelect too often, and that we waited some time
  // (to sample the metrics) from the event last_change_time
  // last_change_time is the time where user changed the size/rate/frame rate
  // (via SetEncodingData).
  int64_t now = clock_->TimeInMilliseconds();
  if ((now - last_qm_update_time_) < kQmMinIntervalMs ||
      (now - last_change_time_) < kQmMinIntervalMs) {
    status = false;
  }

  return status;
}

// Allowing VCM to keep track of incoming frame rate.
void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
  int32_t num = 0;
  int32_t nr_of_frames = 0;
  for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
    if (incoming_frame_times_[num] <= 0 ||
        // don't use data older than 2 s
        now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
      break;
    } else {
      nr_of_frames++;
    }
  }
  if (num > 1) {
    const int64_t diff =
        incoming_frame_times_[0] - incoming_frame_times_[num - 1];
    incoming_frame_rate_ = 0.0;  // No frame rate estimate available.
    if (diff > 0) {
      incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
    }
  }
}

void MediaOptimization::CheckSuspendConditions() {
  // Check conditions for SuspendBelowMinBitrate. |video_target_bitrate_| is in
  // bps.
  if (suspension_enabled_) {
    if (!video_suspended_) {
      // Check if we just went below the threshold.
      if (video_target_bitrate_ < suspension_threshold_bps_) {
        video_suspended_ = true;
      }
    } else {
      // Video is already suspended. Check if we just went over the threshold
      // with a margin.
      if (video_target_bitrate_ >
          suspension_threshold_bps_ + suspension_window_bps_) {
        video_suspended_ = false;
      }
    }
  }
}

}  // namespace media_optimization
}  // namespace webrtc