webrtc/modules/video_coding/media_optimization.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/video_coding/media_optimization.h"

#include <limits>

#include "webrtc/base/logging.h"
#include "webrtc/modules/video_coding/utility/frame_dropper.h"
#include "webrtc/system_wrappers/include/clock.h"

namespace webrtc {
namespace media_optimization {
const int kMsPerSec = 1000;
const int kBitsPerByte = 8;

struct MediaOptimization::EncodedFrameSample {
  EncodedFrameSample(size_t size_bytes,
                     uint32_t timestamp,
                     int64_t time_complete_ms)
      : size_bytes(size_bytes),
        timestamp(timestamp),
        time_complete_ms(time_complete_ms) {}

  size_t size_bytes;
  uint32_t timestamp;
  int64_t time_complete_ms;
};

MediaOptimization::MediaOptimization(Clock* clock)
    : clock_(clock),
      max_bit_rate_(0),
      codec_width_(0),
      codec_height_(0),
      user_frame_rate_(0),
      frame_dropper_(new FrameDropper),
      send_statistics_zero_encode_(0),
      max_payload_size_(1460),
      video_target_bitrate_(0),
      incoming_frame_rate_(0),
      encoded_frame_samples_(),
      avg_sent_framerate_(0),
      num_layers_(0) {
  memset(send_statistics_, 0, sizeof(send_statistics_));
  memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
}

MediaOptimization::~MediaOptimization(void) {
}

void MediaOptimization::Reset() {
  rtc::CritScope lock(&crit_sect_);
  SetEncodingDataInternal(0, 0, 0, 0, 0, 0, max_payload_size_);
  memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
  incoming_frame_rate_ = 0.0;
  frame_dropper_->Reset();
  frame_dropper_->SetRates(0, 0);
  send_statistics_zero_encode_ = 0;
  video_target_bitrate_ = 0;
  codec_width_ = 0;
  codec_height_ = 0;
  user_frame_rate_ = 0;
  encoded_frame_samples_.clear();
  num_layers_ = 1;
}

void MediaOptimization::SetEncodingData(int32_t max_bit_rate,
                                        uint32_t target_bitrate,
                                        uint16_t width,
                                        uint16_t height,
                                        uint32_t frame_rate,
                                        int num_layers,
                                        int32_t mtu) {
  rtc::CritScope lock(&crit_sect_);
  SetEncodingDataInternal(max_bit_rate, frame_rate, target_bitrate, width,
                          height, num_layers, mtu);
}

void MediaOptimization::SetEncodingDataInternal(int32_t max_bit_rate,
                                                uint32_t frame_rate,
                                                uint32_t target_bitrate,
                                                uint16_t width,
                                                uint16_t height,
                                                int num_layers,
                                                int32_t mtu) {
  // Everything codec specific should be reset here since this means the codec
  // has changed.

  max_bit_rate_ = max_bit_rate;
  video_target_bitrate_ = target_bitrate;
  float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
  frame_dropper_->Reset();
  frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate));
  user_frame_rate_ = static_cast<float>(frame_rate);
  codec_width_ = width;
  codec_height_ = height;
  num_layers_ = (num_layers <= 1) ? 1 : num_layers;  // Can also be zero.
  max_payload_size_ = mtu;
}

uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate) {
  rtc::CritScope lock(&crit_sect_);

  video_target_bitrate_ = target_bitrate;

  // Cap target video bitrate to codec maximum.
  if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) {
    video_target_bitrate_ = max_bit_rate_;
  }

  // Update encoding rates following protection settings.
  float target_video_bitrate_kbps =
      static_cast<float>(video_target_bitrate_) / 1000.0f;
  float framerate = incoming_frame_rate_;
  if (framerate == 0.0) {
    // No framerate estimate available, use configured max framerate instead.
    framerate = user_frame_rate_;
  }

  frame_dropper_->SetRates(target_video_bitrate_kbps, framerate);

  return video_target_bitrate_;
}

uint32_t MediaOptimization::InputFrameRate() {
  rtc::CritScope lock(&crit_sect_);
  return InputFrameRateInternal();
}

uint32_t MediaOptimization::InputFrameRateInternal() {
  ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
  uint32_t framerate = static_cast<uint32_t>(std::min<float>(
      std::numeric_limits<uint32_t>::max(), incoming_frame_rate_ + 0.5f));
  return framerate;
}

uint32_t MediaOptimization::SentFrameRate() {
  rtc::CritScope lock(&crit_sect_);
  return SentFrameRateInternal();
}

uint32_t MediaOptimization::SentFrameRateInternal() {
  PurgeOldFrameSamples(clock_->TimeInMilliseconds() - kBitrateAverageWinMs);
  UpdateSentFramerate();
  return avg_sent_framerate_;
}

uint32_t MediaOptimization::SentBitRate() {
  rtc::CritScope lock(&crit_sect_);
  PurgeOldFrameSamples(clock_->TimeInMilliseconds() - kBitrateAverageWinMs);
  size_t sent_bytes = 0;
  for (auto& frame_sample : encoded_frame_samples_) {
    sent_bytes += frame_sample.size_bytes;
  }
  return sent_bytes * kBitsPerByte * kMsPerSec / kBitrateAverageWinMs;
}

int32_t MediaOptimization::UpdateWithEncodedData(
    const EncodedImage& encoded_image) {
  size_t encoded_length = encoded_image._length;
  uint32_t timestamp = encoded_image._timeStamp;
  rtc::CritScope lock(&crit_sect_);
  const int64_t now_ms = clock_->TimeInMilliseconds();
  PurgeOldFrameSamples(now_ms - kBitrateAverageWinMs);
  if (encoded_frame_samples_.size() > 0 &&
      encoded_frame_samples_.back().timestamp == timestamp) {
    // Frames having the same timestamp are generated from the same input
    // frame. We don't want to double count them, but only increment the
    // size_bytes.
    encoded_frame_samples_.back().size_bytes += encoded_length;
    encoded_frame_samples_.back().time_complete_ms = now_ms;
  } else {
    encoded_frame_samples_.push_back(
        EncodedFrameSample(encoded_length, timestamp, now_ms));
  }
  UpdateSentFramerate();
  if (encoded_length > 0) {
    const bool delta_frame = encoded_image._frameType != kVideoFrameKey;
    frame_dropper_->Fill(encoded_length, delta_frame);
  }

  return VCM_OK;
}

void MediaOptimization::EnableFrameDropper(bool enable) {
  rtc::CritScope lock(&crit_sect_);
  frame_dropper_->Enable(enable);
}

bool MediaOptimization::DropFrame() {
  rtc::CritScope lock(&crit_sect_);
  UpdateIncomingFrameRate();
  // Leak appropriate number of bytes.
  frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f));
  return frame_dropper_->DropFrame();
}

void MediaOptimization::UpdateIncomingFrameRate() {
  int64_t now = clock_->TimeInMilliseconds();
  if (incoming_frame_times_[0] == 0) {
    // No shifting if this is the first time.
  } else {
    // Shift all times one step.
    for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
      incoming_frame_times_[i + 1] = incoming_frame_times_[i];
    }
  }
  incoming_frame_times_[0] = now;
  ProcessIncomingFrameRate(now);
}

void MediaOptimization::PurgeOldFrameSamples(int64_t threshold_ms) {
  while (!encoded_frame_samples_.empty()) {
    if (encoded_frame_samples_.front().time_complete_ms < threshold_ms) {
      encoded_frame_samples_.pop_front();
    } else {
      break;
    }
  }
}

void MediaOptimization::UpdateSentFramerate() {
  if (encoded_frame_samples_.size() <= 1) {
    avg_sent_framerate_ = encoded_frame_samples_.size();
    return;
  }
  int denom = encoded_frame_samples_.back().timestamp -
              encoded_frame_samples_.front().timestamp;
  if (denom > 0) {
    avg_sent_framerate_ =
        (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom;
  } else {
    avg_sent_framerate_ = encoded_frame_samples_.size();
  }
}

// Allowing VCM to keep track of incoming frame rate.
void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
  int32_t num = 0;
  int32_t nr_of_frames = 0;
  for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
    if (incoming_frame_times_[num] <= 0 ||
        // don't use data older than 2 s
        now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
      break;
    } else {
      nr_of_frames++;
    }
  }
  if (num > 1) {
    const int64_t diff =
        incoming_frame_times_[0] - incoming_frame_times_[num - 1];
    incoming_frame_rate_ = 0.0;  // No frame rate estimate available.
    if (diff > 0) {
      incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
    }
  }
}
}  // namespace media_optimization
}  // namespace webrtc