Refactor and remove media_optimization::MediaOptimization.
This CL removes MediaOptmization and folds some of its functionality
into VideoStreamEncoder.
The FPS tracking is now handled by a RateStatistics instance. Frame
dropping is still handled by FrameDropper. Both of these now live
directly in VideoStreamEncoder.
There is no intended change in behavior from this CL, but due to a new
way of measuring frame rate, some minor perf changes can be expected.
A small change in behavior is that OnBitrateUpdated is now called
directly rather than on the next frame. Since both encoding frame and
setting rate allocations happen on the encoder worker thread, there's
really no reason to cache bitrates and wait until the next frame.
An edge case though is that if a new bitrate is set before the first
frame, we must remember that bitrate and then apply it after the video
bitrate allocator has been first created.
In addition to existing unit tests, manual tests have been used to
confirm that frame dropping works as expected with misbehaving encoders.
Bug: webrtc:10164
Change-Id: I7ee9c8d3c4f2bcf23c8c420310b05a4d35d94744
Reviewed-on: https://webrtc-review.googlesource.com/c/115620
Commit-Queue: Erik Språng <sprang@webrtc.org>
Reviewed-by: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26147}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 9f6d9f3..dbfa0dc 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -133,8 +133,6 @@
"jitter_estimator.h",
"media_opt_util.cc",
"media_opt_util.h",
- "media_optimization.cc",
- "media_optimization.h",
"nack_fec_tables.h",
"packet_buffer.cc",
"packet_buffer.h",
diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc
index 9dd07bc..2d992ef 100644
--- a/modules/video_coding/generic_encoder.cc
+++ b/modules/video_coding/generic_encoder.cc
@@ -22,7 +22,6 @@
#include "api/video/video_timing.h"
#include "modules/include/module_common_types_public.h"
#include "modules/video_coding/include/video_coding_defines.h"
-#include "modules/video_coding/media_optimization.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/alr_experiment.h"
#include "rtc_base/logging.h"
@@ -46,6 +45,7 @@
: encoder_(encoder),
vcm_encoded_frame_callback_(encoded_frame_callback),
internal_source_(internal_source),
+ input_frame_rate_(0),
streams_or_svc_num_(0),
codec_type_(VideoCodecType::kVideoCodecGeneric) {}
@@ -103,38 +103,34 @@
return encoder_->Encode(frame, codec_specific, &frame_types);
}
-void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) {
+void VCMGenericEncoder::SetEncoderParameters(
+ const VideoBitrateAllocation& target_bitrate,
+ uint32_t input_frame_rate) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
bool rates_have_changed;
{
rtc::CritScope lock(¶ms_lock_);
- rates_have_changed =
- params.target_bitrate != encoder_params_.target_bitrate ||
- params.input_frame_rate != encoder_params_.input_frame_rate;
- encoder_params_ = params;
+ rates_have_changed = target_bitrate != bitrate_allocation_ ||
+ input_frame_rate != input_frame_rate_;
+ bitrate_allocation_ = target_bitrate;
+ input_frame_rate_ = input_frame_rate;
}
if (rates_have_changed) {
- int res = encoder_->SetRateAllocation(params.target_bitrate,
- params.input_frame_rate);
+ int res = encoder_->SetRateAllocation(target_bitrate, input_frame_rate);
if (res != 0) {
RTC_LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
- << params.target_bitrate.get_sum_bps()
- << ", framerate = " << params.input_frame_rate
+ << target_bitrate.get_sum_bps()
+ << ", framerate = " << input_frame_rate
<< "): " << res;
}
- vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate);
+ vcm_encoded_frame_callback_->OnFrameRateChanged(input_frame_rate);
for (size_t i = 0; i < streams_or_svc_num_; ++i) {
vcm_encoded_frame_callback_->OnTargetBitrateChanged(
- params.target_bitrate.GetSpatialLayerSum(i) / 8, i);
+ target_bitrate.GetSpatialLayerSum(i) / 8, i);
}
}
}
-EncoderParameters VCMGenericEncoder::GetEncoderParameters() const {
- rtc::CritScope lock(¶ms_lock_);
- return encoder_params_;
-}
-
int32_t VCMGenericEncoder::RequestFrame(
const std::vector<FrameType>& frame_types) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
@@ -164,11 +160,9 @@
}
VCMEncodedFrameCallback::VCMEncodedFrameCallback(
- EncodedImageCallback* post_encode_callback,
- media_optimization::MediaOptimization* media_opt)
+ EncodedImageCallback* post_encode_callback)
: internal_source_(false),
post_encode_callback_(post_encode_callback),
- media_opt_(media_opt),
framerate_(1),
last_timing_frame_time_ms_(-1),
timing_frames_thresholds_({-1, 0}),
@@ -406,20 +400,8 @@
RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
&image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
- Result result = post_encode_callback_->OnEncodedImage(
- image_copy, codec_specific, fragmentation_header);
- if (result.error != Result::OK)
- return result;
-
- if (media_opt_) {
- media_opt_->UpdateWithEncodedData(image_copy._length,
- image_copy._frameType);
- if (internal_source_) {
- // Signal to encoder to drop next frame.
- result.drop_next_frame = media_opt_->DropFrame();
- }
- }
- return result;
+ return post_encode_callback_->OnEncodedImage(image_copy, codec_specific,
+ fragmentation_header);
}
} // namespace webrtc
diff --git a/modules/video_coding/generic_encoder.h b/modules/video_coding/generic_encoder.h
index 1f4ade2..463f98a 100644
--- a/modules/video_coding/generic_encoder.h
+++ b/modules/video_coding/generic_encoder.h
@@ -23,33 +23,9 @@
namespace webrtc {
-namespace media_optimization {
-class MediaOptimization;
-} // namespace media_optimization
-
-struct EncoderParameters {
- EncoderParameters() : total_bitrate(DataRate::Zero()), input_frame_rate(0) {}
- EncoderParameters(DataRate total_bitrate,
- const VideoBitrateAllocation& allocation,
- uint32_t framerate)
- : total_bitrate(total_bitrate),
- target_bitrate(allocation),
- input_frame_rate(framerate) {}
-
- // Total bitrate allocated for this encoder.
- DataRate total_bitrate;
- // The bitrate allocation, across spatial and/or temporal layers. Note that
- // the sum of these might be less than |total_bitrate| if the allocator has
- // capped the bitrate for some configuration.
- VideoBitrateAllocation target_bitrate;
- // The input frame rate to the encoder in fps, measured in the video sender.
- uint32_t input_frame_rate;
-};
-
class VCMEncodedFrameCallback : public EncodedImageCallback {
public:
- VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback,
- media_optimization::MediaOptimization* media_opt);
+ explicit VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback);
~VCMEncodedFrameCallback() override;
// Implements EncodedImageCallback.
@@ -100,7 +76,6 @@
rtc::CriticalSection timing_params_lock_;
bool internal_source_;
EncodedImageCallback* const post_encode_callback_;
- media_optimization::MediaOptimization* const media_opt_;
struct EncodeStartTimeRecord {
EncodeStartTimeRecord(uint32_t timestamp,
@@ -153,8 +128,8 @@
const CodecSpecificInfo* codec_specific,
const std::vector<FrameType>& frame_types);
- void SetEncoderParameters(const EncoderParameters& params);
- EncoderParameters GetEncoderParameters() const;
+ void SetEncoderParameters(const VideoBitrateAllocation& target_bitrate,
+ uint32_t input_frame_rate);
int32_t RequestFrame(const std::vector<FrameType>& frame_types);
bool InternalSource() const;
@@ -167,7 +142,8 @@
VCMEncodedFrameCallback* const vcm_encoded_frame_callback_;
const bool internal_source_;
rtc::CriticalSection params_lock_;
- EncoderParameters encoder_params_ RTC_GUARDED_BY(params_lock_);
+ VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(params_lock_);
+ uint32_t input_frame_rate_ RTC_GUARDED_BY(params_lock_);
size_t streams_or_svc_num_ RTC_GUARDED_BY(race_checker_);
VideoCodecType codec_type_ RTC_GUARDED_BY(race_checker_);
};
diff --git a/modules/video_coding/generic_encoder_unittest.cc b/modules/video_coding/generic_encoder_unittest.cc
index 4d5df5b..9491b5a 100644
--- a/modules/video_coding/generic_encoder_unittest.cc
+++ b/modules/video_coding/generic_encoder_unittest.cc
@@ -76,7 +76,7 @@
const int num_streams,
const int num_frames) {
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
const size_t kFramerate = 30;
callback.SetTimingFramesThresholds(
{delay_ms, kDefaultOutlierFrameSizePercent});
@@ -192,7 +192,7 @@
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
callback.SetTimingFramesThresholds(thresholds);
@@ -223,7 +223,7 @@
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
callback.SetInternalSource(true);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
@@ -258,7 +258,7 @@
const int64_t kTimestampMs4 = 47721870;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs1;
@@ -293,7 +293,7 @@
const int64_t kTimestampMs = 123456;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs; // Incorrect timesetamp.
diff --git a/modules/video_coding/media_optimization.cc b/modules/video_coding/media_optimization.cc
deleted file mode 100644
index f74e5eb..0000000
--- a/modules/video_coding/media_optimization.cc
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "modules/video_coding/media_optimization.h"
-
-#include <string.h>
-#include <algorithm>
-#include <limits>
-
-#include "modules/video_coding/utility/frame_dropper.h"
-#include "system_wrappers/include/clock.h"
-
-namespace webrtc {
-namespace media_optimization {
-namespace {
-const int64_t kFrameHistoryWinMs = 2000;
-} // namespace
-
-MediaOptimization::MediaOptimization(Clock* clock)
- : clock_(clock),
- max_bit_rate_(0),
- max_frame_rate_(0),
- frame_dropper_(),
- incoming_frame_rate_(0) {
- memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
- frame_dropper_.SetRates(0, 0);
-}
-
-MediaOptimization::~MediaOptimization() = default;
-
-void MediaOptimization::SetEncodingData(int32_t max_bit_rate,
- uint32_t target_bitrate,
- uint32_t max_frame_rate) {
- rtc::CritScope lock(&crit_sect_);
- // Everything codec specific should be reset here since the codec has changed.
- max_bit_rate_ = max_bit_rate;
- max_frame_rate_ = static_cast<float>(max_frame_rate);
- float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
- frame_dropper_.Reset();
- frame_dropper_.SetRates(target_bitrate_kbps, max_frame_rate_);
-}
-
-uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate) {
- rtc::CritScope lock(&crit_sect_);
-
- // Cap target video bitrate to codec maximum.
- int video_target_bitrate = target_bitrate;
- if (max_bit_rate_ > 0 && video_target_bitrate > max_bit_rate_) {
- video_target_bitrate = max_bit_rate_;
- }
- float target_video_bitrate_kbps =
- static_cast<float>(video_target_bitrate) / 1000.0f;
-
- float framerate = incoming_frame_rate_;
- if (framerate == 0.0) {
- // No framerate estimate available, use configured max framerate instead.
- framerate = max_frame_rate_;
- }
-
- frame_dropper_.SetRates(target_video_bitrate_kbps, framerate);
-
- return video_target_bitrate;
-}
-
-uint32_t MediaOptimization::InputFrameRate() {
- rtc::CritScope lock(&crit_sect_);
- return InputFrameRateInternal();
-}
-
-uint32_t MediaOptimization::InputFrameRateInternal() {
- ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
- uint32_t framerate = static_cast<uint32_t>(std::min<float>(
- std::numeric_limits<uint32_t>::max(), incoming_frame_rate_ + 0.5f));
- return framerate;
-}
-
-void MediaOptimization::UpdateWithEncodedData(
- const size_t encoded_image_length,
- const FrameType encoded_image_frametype) {
- size_t encoded_length = encoded_image_length;
- rtc::CritScope lock(&crit_sect_);
- if (encoded_length > 0) {
- const bool delta_frame = encoded_image_frametype != kVideoFrameKey;
- frame_dropper_.Fill(encoded_length, delta_frame);
- }
-}
-
-void MediaOptimization::EnableFrameDropper(bool enable) {
- rtc::CritScope lock(&crit_sect_);
- frame_dropper_.Enable(enable);
-}
-
-bool MediaOptimization::DropFrame() {
- rtc::CritScope lock(&crit_sect_);
- UpdateIncomingFrameRate();
- // Leak appropriate number of bytes.
- frame_dropper_.Leak(static_cast<uint32_t>(InputFrameRateInternal() + 0.5f));
- return frame_dropper_.DropFrame();
-}
-
-void MediaOptimization::UpdateIncomingFrameRate() {
- int64_t now = clock_->TimeInMilliseconds();
- if (incoming_frame_times_[0] == 0) {
- // No shifting if this is the first time.
- } else {
- // Shift all times one step.
- for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
- incoming_frame_times_[i + 1] = incoming_frame_times_[i];
- }
- }
- incoming_frame_times_[0] = now;
- ProcessIncomingFrameRate(now);
-}
-
-// Allowing VCM to keep track of incoming frame rate.
-void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
- int32_t num = 0;
- int32_t nr_of_frames = 0;
- for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
- if (incoming_frame_times_[num] <= 0 ||
- // Don't use data older than 2 s.
- now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
- break;
- } else {
- nr_of_frames++;
- }
- }
- if (num > 1) {
- const int64_t diff =
- incoming_frame_times_[0] - incoming_frame_times_[num - 1];
- incoming_frame_rate_ = 0.0; // No frame rate estimate available.
- if (diff > 0) {
- incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
- }
- }
-}
-} // namespace media_optimization
-} // namespace webrtc
diff --git a/modules/video_coding/media_optimization.h b/modules/video_coding/media_optimization.h
deleted file mode 100644
index 0b6b82f..0000000
--- a/modules/video_coding/media_optimization.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
-#define MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
-
-#include <memory>
-
-#include "modules/include/module_common_types.h"
-#include "modules/video_coding/include/video_coding.h"
-#include "modules/video_coding/media_opt_util.h"
-#include "modules/video_coding/utility/frame_dropper.h"
-#include "rtc_base/criticalsection.h"
-
-namespace webrtc {
-
-class Clock;
-
-namespace media_optimization {
-
-class MediaOptimization {
- public:
- explicit MediaOptimization(Clock* clock);
- ~MediaOptimization();
-
- // Informs media optimization of initial encoding state.
- // TODO(perkj): Deprecate SetEncodingData once its not used for stats in
- // VideoStreamEncoder.
- void SetEncodingData(int32_t max_bit_rate,
- uint32_t bit_rate,
- uint32_t max_frame_rate);
-
- // Sets target rates for the encoder given the channel parameters.
- // Input: |target bitrate| - the encoder target bitrate in bits/s.
- uint32_t SetTargetRates(uint32_t target_bitrate);
-
- void EnableFrameDropper(bool enable);
- bool DropFrame();
-
- // Informs Media Optimization of encoded output.
- // TODO(perkj): Deprecate SetEncodingData once its not used for stats in
- // VideoStreamEncoder.
- void UpdateWithEncodedData(const size_t encoded_image_length,
- const FrameType encoded_image_frametype);
-
- // InputFrameRate 0 = no frame rate estimate available.
- uint32_t InputFrameRate();
-
- private:
- enum { kFrameCountHistorySize = 90 };
-
- void UpdateIncomingFrameRate() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
- void ProcessIncomingFrameRate(int64_t now)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
- uint32_t InputFrameRateInternal() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
-
- // Protect all members.
- rtc::CriticalSection crit_sect_;
-
- Clock* const clock_ RTC_GUARDED_BY(crit_sect_);
- int32_t max_bit_rate_ RTC_GUARDED_BY(crit_sect_);
- float max_frame_rate_ RTC_GUARDED_BY(crit_sect_);
- FrameDropper frame_dropper_ RTC_GUARDED_BY(crit_sect_);
- float incoming_frame_rate_ RTC_GUARDED_BY(crit_sect_);
- int64_t incoming_frame_times_[kFrameCountHistorySize] RTC_GUARDED_BY(
- crit_sect_);
-};
-} // namespace media_optimization
-} // namespace webrtc
-
-#endif // MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
diff --git a/modules/video_coding/video_coding_impl.h b/modules/video_coding/video_coding_impl.h
index e7fb74c..08b0753 100644
--- a/modules/video_coding/video_coding_impl.h
+++ b/modules/video_coding/video_coding_impl.h
@@ -24,7 +24,6 @@
#include "modules/video_coding/generic_decoder.h"
#include "modules/video_coding/generic_encoder.h"
#include "modules/video_coding/jitter_buffer.h"
-#include "modules/video_coding/media_optimization.h"
#include "modules/video_coding/receiver.h"
#include "modules/video_coding/timing.h"
#include "rtc_base/onetimeevent.h"
@@ -63,7 +62,6 @@
typedef VideoCodingModule::SenderNackMode SenderNackMode;
VideoSender(Clock* clock, EncodedImageCallback* post_encode_callback);
-
~VideoSender();
// Register the send codec to be used.
@@ -75,60 +73,27 @@
void RegisterExternalEncoder(VideoEncoder* externalEncoder,
bool internalSource);
- // Update the channel parameters based on new rates and rtt. This will also
- // cause an immediate call to VideoEncoder::SetRateAllocation().
- int32_t SetChannelParameters(
- uint32_t target_bitrate_bps,
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback);
-
- // Updates the channel parameters with a new bitrate allocation, but using the
- // current targit_bitrate, loss rate and rtt. That is, the distribution or
- // caps may be updated to a change to a new VideoCodec or allocation mode.
- // The new parameters will be stored as pending EncoderParameters, and the
- // encoder will only be updated on the next frame.
- void UpdateChannelParameters(
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback);
+ // Update the the encoder with new bitrate allocation and framerate.
+ int32_t SetChannelParameters(const VideoBitrateAllocation& bitrate_allocation,
+ uint32_t framerate_fps);
int32_t AddVideoFrame(const VideoFrame& videoFrame,
const CodecSpecificInfo* codecSpecificInfo,
absl::optional<VideoEncoder::EncoderInfo> encoder_info);
int32_t IntraFrameRequest(size_t stream_index);
- int32_t EnableFrameDropper(bool enable);
private:
- EncoderParameters UpdateEncoderParameters(
- const EncoderParameters& params,
- VideoBitrateAllocator* bitrate_allocator,
- uint32_t target_bitrate_bps);
- void SetEncoderParameters(EncoderParameters params, bool has_internal_source)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_);
- VideoBitrateAllocation GetAllocation(
- uint32_t bitrate_bps,
- uint32_t framerate_fps,
- VideoBitrateAllocator* bitrate_allocator) const;
-
rtc::CriticalSection encoder_crit_;
VCMGenericEncoder* _encoder;
- media_optimization::MediaOptimization _mediaOpt;
VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_);
- EncodedImageCallback* const post_encode_callback_;
VCMEncoderDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_);
- // If frame dropper is not force disabled, frame dropping might still be
- // disabled if VideoEncoder::GetEncoderInfo() indicates that the encoder has a
- // trusted rate controller. This is determined on a per-frame basis, as the
- // encoder behavior might dynamically change.
- bool force_disable_frame_dropper_ RTC_GUARDED_BY(encoder_crit_);
-
// Must be accessed on the construction thread of VideoSender.
VideoCodec current_codec_;
rtc::SequencedTaskChecker sequenced_checker_;
rtc::CriticalSection params_crit_;
- EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_);
bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_);
std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_);
};
diff --git a/modules/video_coding/video_sender.cc b/modules/video_coding/video_sender.cc
index aea26f7..fe959b8 100644
--- a/modules/video_coding/video_sender.cc
+++ b/modules/video_coding/video_sender.cc
@@ -25,7 +25,6 @@
#include "modules/video_coding/include/video_coding_defines.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/internal_defines.h"
-#include "modules/video_coding/media_optimization.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
#include "modules/video_coding/video_coding_impl.h"
#include "rtc_base/checks.h"
@@ -39,20 +38,11 @@
namespace webrtc {
namespace vcm {
-namespace {
-
-constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
-
-} // namespace
-
VideoSender::VideoSender(Clock* clock,
EncodedImageCallback* post_encode_callback)
: _encoder(nullptr),
- _mediaOpt(clock),
- _encodedFrameCallback(post_encode_callback, &_mediaOpt),
- post_encode_callback_(post_encode_callback),
+ _encodedFrameCallback(post_encode_callback),
_codecDataBase(&_encodedFrameCallback),
- force_disable_frame_dropper_(false),
current_codec_(),
encoder_has_internal_source_(false),
next_frame_types_(1, kVideoFrameDelta) {
@@ -93,27 +83,6 @@
// SetSendCodec succeeded, _encoder should be set.
RTC_DCHECK(_encoder);
- int numLayers;
- if (sendCodec->codecType == kVideoCodecVP8) {
- numLayers = sendCodec->VP8().numberOfTemporalLayers;
- } else if (sendCodec->codecType == kVideoCodecVP9) {
- numLayers = sendCodec->VP9().numberOfTemporalLayers;
- } else if (sendCodec->codecType == kVideoCodecGeneric &&
- sendCodec->numberOfSimulcastStreams > 0) {
- // This is mainly for unit testing, disabling frame dropping.
- // TODO(sprang): Add a better way to disable frame dropping.
- numLayers = sendCodec->simulcastStream[0].numberOfTemporalLayers;
- } else {
- numLayers = 1;
- }
-
- // Force-disable frame dropper if either:
- // * We have screensharing with layers.
- // * "WebRTC-FrameDropper" field trial is "Disabled".
- force_disable_frame_dropper_ =
- field_trial::IsDisabled(kFrameDropperFieldTrial) ||
- (numLayers > 1 && sendCodec->mode == VideoCodecMode::kScreensharing);
-
{
rtc::CritScope cs(¶ms_crit_);
next_frame_types_.clear();
@@ -128,9 +97,6 @@
<< " start bitrate " << sendCodec->startBitrate
<< " max frame rate " << sendCodec->maxFramerate
<< " max payload size " << maxPayloadSize;
- _mediaOpt.SetEncodingData(sendCodec->maxBitrate * 1000,
- sendCodec->startBitrate * 1000,
- sendCodec->maxFramerate);
return VCM_OK;
}
@@ -156,149 +122,58 @@
internalSource);
}
-EncoderParameters VideoSender::UpdateEncoderParameters(
- const EncoderParameters& params,
- VideoBitrateAllocator* bitrate_allocator,
- uint32_t target_bitrate_bps) {
- uint32_t video_target_rate_bps = _mediaOpt.SetTargetRates(target_bitrate_bps);
- uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
- if (input_frame_rate == 0)
- input_frame_rate = current_codec_.maxFramerate;
-
- EncoderParameters new_encoder_params = {
- DataRate::bps(target_bitrate_bps),
- GetAllocation(video_target_rate_bps, input_frame_rate, bitrate_allocator),
- input_frame_rate};
- return new_encoder_params;
-}
-
-VideoBitrateAllocation VideoSender::GetAllocation(
- uint32_t bitrate_bps,
- uint32_t framerate_fps,
- VideoBitrateAllocator* bitrate_allocator) const {
- VideoBitrateAllocation bitrate_allocation;
- // Only call allocators if bitrate > 0 (ie, not suspended), otherwise they
- // might cap the bitrate to the min bitrate configured.
- if (bitrate_bps > 0) {
- if (bitrate_allocator) {
- bitrate_allocation =
- bitrate_allocator->GetAllocation(bitrate_bps, framerate_fps);
- } else {
- DefaultVideoBitrateAllocator default_allocator(current_codec_);
- bitrate_allocation =
- default_allocator.GetAllocation(bitrate_bps, framerate_fps);
- }
- }
- return bitrate_allocation;
-}
-
-void VideoSender::UpdateChannelParameters(
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback) {
- VideoBitrateAllocation target_rate;
- {
- rtc::CritScope cs(¶ms_crit_);
- encoder_params_ =
- UpdateEncoderParameters(encoder_params_, bitrate_allocator,
- encoder_params_.total_bitrate.bps());
- target_rate = encoder_params_.target_bitrate;
- }
- if (bitrate_updated_callback && target_rate.get_sum_bps() > 0) {
- bitrate_updated_callback->OnBitrateAllocationUpdated(target_rate);
- }
-}
-
int32_t VideoSender::SetChannelParameters(
- uint32_t target_bitrate_bps,
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback) {
- EncoderParameters encoder_params;
- encoder_params = UpdateEncoderParameters(encoder_params, bitrate_allocator,
- target_bitrate_bps);
- if (bitrate_updated_callback && target_bitrate_bps > 0) {
- bitrate_updated_callback->OnBitrateAllocationUpdated(
- encoder_params.target_bitrate);
- }
-
+ const VideoBitrateAllocation& bitrate_allocation,
+ uint32_t framerate_fps) {
bool encoder_has_internal_source;
{
rtc::CritScope cs(¶ms_crit_);
- encoder_params_ = encoder_params;
encoder_has_internal_source = encoder_has_internal_source_;
}
- // For encoders with internal sources, we need to tell the encoder directly,
- // instead of waiting for an AddVideoFrame that will never come (internal
- // source encoders don't get input frames).
- if (encoder_has_internal_source) {
+ {
rtc::CritScope cs(&encoder_crit_);
if (_encoder) {
- SetEncoderParameters(encoder_params, encoder_has_internal_source);
+ // |target_bitrate == 0 | means that the network is down or the send pacer
+ // is full. We currently only report this if the encoder has an internal
+ // source. If the encoder does not have an internal source, higher levels
+ // are expected to not call AddVideoFrame. We do this since its unclear
+ // how current encoder implementations behave when given a zero target
+ // bitrate.
+ // TODO(perkj): Make sure all known encoder implementations handle zero
+ // target bitrate and remove this check.
+ if (!encoder_has_internal_source &&
+ bitrate_allocation.get_sum_bps() == 0) {
+ return VCM_OK;
+ }
+
+ if (framerate_fps == 0) {
+ // No frame rate estimate available, use default.
+ framerate_fps = current_codec_.maxFramerate;
+ }
+ if (_encoder != nullptr)
+ _encoder->SetEncoderParameters(bitrate_allocation, framerate_fps);
}
}
return VCM_OK;
}
-void VideoSender::SetEncoderParameters(EncoderParameters params,
- bool has_internal_source) {
- // |target_bitrate == 0 | means that the network is down or the send pacer is
- // full. We currently only report this if the encoder has an internal source.
- // If the encoder does not have an internal source, higher levels are expected
- // to not call AddVideoFrame. We do this since its unclear how current
- // encoder implementations behave when given a zero target bitrate.
- // TODO(perkj): Make sure all known encoder implementations handle zero
- // target bitrate and remove this check.
- if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0)
- return;
-
- if (params.input_frame_rate == 0) {
- // No frame rate estimate available, use default.
- params.input_frame_rate = current_codec_.maxFramerate;
- }
- if (_encoder != nullptr)
- _encoder->SetEncoderParameters(params);
-}
-
// Add one raw video frame to the encoder, blocking.
int32_t VideoSender::AddVideoFrame(
const VideoFrame& videoFrame,
const CodecSpecificInfo* codecSpecificInfo,
absl::optional<VideoEncoder::EncoderInfo> encoder_info) {
- EncoderParameters encoder_params;
std::vector<FrameType> next_frame_types;
bool encoder_has_internal_source = false;
{
rtc::CritScope lock(¶ms_crit_);
- encoder_params = encoder_params_;
next_frame_types = next_frame_types_;
encoder_has_internal_source = encoder_has_internal_source_;
}
rtc::CritScope lock(&encoder_crit_);
if (_encoder == nullptr)
return VCM_UNINITIALIZED;
- SetEncoderParameters(encoder_params, encoder_has_internal_source);
- if (!encoder_info) {
- encoder_info = _encoder->GetEncoderInfo();
- }
-
- // Frame dropping is enabled iff frame dropping has been requested, and
- // frame dropping is not force-disabled, and rate controller is not trusted.
- const bool frame_dropping_enabled =
- !force_disable_frame_dropper_ &&
- !encoder_info->has_trusted_rate_controller;
- _mediaOpt.EnableFrameDropper(frame_dropping_enabled);
-
- if (_mediaOpt.DropFrame()) {
- RTC_LOG(LS_VERBOSE) << "Drop Frame: "
- << "target bitrate "
- << encoder_params.target_bitrate.get_sum_bps()
- << ", input frame rate "
- << encoder_params.input_frame_rate;
- post_encode_callback_->OnDroppedFrame(
- EncodedImageCallback::DropReason::kDroppedByMediaOptimizations);
- return VCM_OK;
- }
// TODO(pbos): Make sure setting send codec is synchronized with video
// processing so frame size always matches.
if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
diff --git a/modules/video_coding/video_sender_unittest.cc b/modules/video_coding/video_sender_unittest.cc
index 986bd3f..a9c9e45 100644
--- a/modules/video_coding/video_sender_unittest.cc
+++ b/modules/video_coding/video_sender_unittest.cc
@@ -319,14 +319,14 @@
SetRateAllocation(new_rate_allocation, settings_.maxFramerate))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_bitrate_kbps * 1000, rate_allocator_.get(),
- nullptr);
+ sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
AddFrame();
clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
// Expect no call to encoder_.SetRates if the new bitrate is zero.
EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0);
- sender_->SetChannelParameters(0, rate_allocator_.get(), nullptr);
+ sender_->SetChannelParameters(VideoBitrateAllocation(),
+ settings_.maxFramerate);
AddFrame();
}
@@ -363,8 +363,7 @@
EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_bitrate_kbps * 1000, rate_allocator_.get(),
- nullptr);
+ sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
}
TEST_F(TestVideoSenderWithMockEncoder,
@@ -375,8 +374,9 @@
// Expect initial call to SetChannelParameters. Rates are initialized through
// InitEncode and expects no additional call before the framerate (or bitrate)
// updates.
- sender_->SetChannelParameters(settings_.startBitrate * 1000,
- rate_allocator_.get(), nullptr);
+ sender_->SetChannelParameters(
+ rate_allocator_->GetAllocation(settings_.startBitrate * 1000, kInputFps),
+ kInputFps);
while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
AddFrame();
clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
@@ -390,8 +390,7 @@
EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_bitrate_bps, rate_allocator_.get(),
- nullptr);
+ sender_->SetChannelParameters(new_rate_allocation, kInputFps);
AddFrame();
}
@@ -439,12 +438,13 @@
AddFrame();
// SetChannelParameters needs to be called frequently to propagate
// framerate from the media optimization into the encoder.
- // Note: SetChannelParameters fails if less than 2 frames are in the
- // buffer since it will fail to calculate the framerate.
+ const VideoBitrateAllocation bitrate_allocation =
+ rate_allocator_->GetAllocation(available_bitrate_kbps_ * 1000,
+ static_cast<uint32_t>(framerate));
if (i != 0) {
- EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
- available_bitrate_kbps_ * 1000,
- rate_allocator_.get(), nullptr));
+ EXPECT_EQ(VCM_OK,
+ sender_->SetChannelParameters(
+ bitrate_allocation, static_cast<uint32_t>(framerate)));
}
}
}