Revert "Refactor and remove media_optimization::MediaOptimization."
This reverts commit 07276e4f89a93b1479d7aeefa53b4fc32daf001b.
Reason for revert: Speculative revert due to downstream crashes.
Original change's description:
> Refactor and remove media_optimization::MediaOptimization.
>
> This CL removes MediaOptmization and folds some of its functionality
> into VideoStreamEncoder.
>
> The FPS tracking is now handled by a RateStatistics instance. Frame
> dropping is still handled by FrameDropper. Both of these now live
> directly in VideoStreamEncoder.
> There is no intended change in behavior from this CL, but due to a new
> way of measuring frame rate, some minor perf changes can be expected.
>
> A small change in behavior is that OnBitrateUpdated is now called
> directly rather than on the next frame. Since both encoding frame and
> setting rate allocations happen on the encoder worker thread, there's
> really no reason to cache bitrates and wait until the next frame.
> An edge case though is that if a new bitrate is set before the first
> frame, we must remember that bitrate and then apply it after the video
> bitrate allocator has been first created.
>
> In addition to existing unit tests, manual tests have been used to
> confirm that frame dropping works as expected with misbehaving encoders.
>
> Bug: webrtc:10164
> Change-Id: I7ee9c8d3c4f2bcf23c8c420310b05a4d35d94744
> Reviewed-on: https://webrtc-review.googlesource.com/c/115620
> Commit-Queue: Erik Språng <sprang@webrtc.org>
> Reviewed-by: Niels Moller <nisse@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#26147}
TBR=nisse@webrtc.org,sprang@webrtc.org
# Not skipping CQ checks because original CL landed > 1 day ago.
Bug: webrtc:10164
Change-Id: Ie0dae19dd012bc09e793c9661a45823fd760c20c
Reviewed-on: https://webrtc-review.googlesource.com/c/116780
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26191}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index dbfa0dc..9f6d9f3 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -133,6 +133,8 @@
"jitter_estimator.h",
"media_opt_util.cc",
"media_opt_util.h",
+ "media_optimization.cc",
+ "media_optimization.h",
"nack_fec_tables.h",
"packet_buffer.cc",
"packet_buffer.h",
diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc
index 2d992ef..9dd07bc 100644
--- a/modules/video_coding/generic_encoder.cc
+++ b/modules/video_coding/generic_encoder.cc
@@ -22,6 +22,7 @@
#include "api/video/video_timing.h"
#include "modules/include/module_common_types_public.h"
#include "modules/video_coding/include/video_coding_defines.h"
+#include "modules/video_coding/media_optimization.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/alr_experiment.h"
#include "rtc_base/logging.h"
@@ -45,7 +46,6 @@
: encoder_(encoder),
vcm_encoded_frame_callback_(encoded_frame_callback),
internal_source_(internal_source),
- input_frame_rate_(0),
streams_or_svc_num_(0),
codec_type_(VideoCodecType::kVideoCodecGeneric) {}
@@ -103,34 +103,38 @@
return encoder_->Encode(frame, codec_specific, &frame_types);
}
-void VCMGenericEncoder::SetEncoderParameters(
- const VideoBitrateAllocation& target_bitrate,
- uint32_t input_frame_rate) {
+void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
bool rates_have_changed;
{
rtc::CritScope lock(¶ms_lock_);
- rates_have_changed = target_bitrate != bitrate_allocation_ ||
- input_frame_rate != input_frame_rate_;
- bitrate_allocation_ = target_bitrate;
- input_frame_rate_ = input_frame_rate;
+ rates_have_changed =
+ params.target_bitrate != encoder_params_.target_bitrate ||
+ params.input_frame_rate != encoder_params_.input_frame_rate;
+ encoder_params_ = params;
}
if (rates_have_changed) {
- int res = encoder_->SetRateAllocation(target_bitrate, input_frame_rate);
+ int res = encoder_->SetRateAllocation(params.target_bitrate,
+ params.input_frame_rate);
if (res != 0) {
RTC_LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
- << target_bitrate.get_sum_bps()
- << ", framerate = " << input_frame_rate
+ << params.target_bitrate.get_sum_bps()
+ << ", framerate = " << params.input_frame_rate
<< "): " << res;
}
- vcm_encoded_frame_callback_->OnFrameRateChanged(input_frame_rate);
+ vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate);
for (size_t i = 0; i < streams_or_svc_num_; ++i) {
vcm_encoded_frame_callback_->OnTargetBitrateChanged(
- target_bitrate.GetSpatialLayerSum(i) / 8, i);
+ params.target_bitrate.GetSpatialLayerSum(i) / 8, i);
}
}
}
+EncoderParameters VCMGenericEncoder::GetEncoderParameters() const {
+ rtc::CritScope lock(¶ms_lock_);
+ return encoder_params_;
+}
+
int32_t VCMGenericEncoder::RequestFrame(
const std::vector<FrameType>& frame_types) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
@@ -160,9 +164,11 @@
}
VCMEncodedFrameCallback::VCMEncodedFrameCallback(
- EncodedImageCallback* post_encode_callback)
+ EncodedImageCallback* post_encode_callback,
+ media_optimization::MediaOptimization* media_opt)
: internal_source_(false),
post_encode_callback_(post_encode_callback),
+ media_opt_(media_opt),
framerate_(1),
last_timing_frame_time_ms_(-1),
timing_frames_thresholds_({-1, 0}),
@@ -400,8 +406,20 @@
RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
&image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
- return post_encode_callback_->OnEncodedImage(image_copy, codec_specific,
- fragmentation_header);
+ Result result = post_encode_callback_->OnEncodedImage(
+ image_copy, codec_specific, fragmentation_header);
+ if (result.error != Result::OK)
+ return result;
+
+ if (media_opt_) {
+ media_opt_->UpdateWithEncodedData(image_copy._length,
+ image_copy._frameType);
+ if (internal_source_) {
+ // Signal to encoder to drop next frame.
+ result.drop_next_frame = media_opt_->DropFrame();
+ }
+ }
+ return result;
}
} // namespace webrtc
diff --git a/modules/video_coding/generic_encoder.h b/modules/video_coding/generic_encoder.h
index 463f98a..1f4ade2 100644
--- a/modules/video_coding/generic_encoder.h
+++ b/modules/video_coding/generic_encoder.h
@@ -23,9 +23,33 @@
namespace webrtc {
+namespace media_optimization {
+class MediaOptimization;
+} // namespace media_optimization
+
+struct EncoderParameters {
+ EncoderParameters() : total_bitrate(DataRate::Zero()), input_frame_rate(0) {}
+ EncoderParameters(DataRate total_bitrate,
+ const VideoBitrateAllocation& allocation,
+ uint32_t framerate)
+ : total_bitrate(total_bitrate),
+ target_bitrate(allocation),
+ input_frame_rate(framerate) {}
+
+ // Total bitrate allocated for this encoder.
+ DataRate total_bitrate;
+ // The bitrate allocation, across spatial and/or temporal layers. Note that
+ // the sum of these might be less than |total_bitrate| if the allocator has
+ // capped the bitrate for some configuration.
+ VideoBitrateAllocation target_bitrate;
+ // The input frame rate to the encoder in fps, measured in the video sender.
+ uint32_t input_frame_rate;
+};
+
class VCMEncodedFrameCallback : public EncodedImageCallback {
public:
- explicit VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback);
+ VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback,
+ media_optimization::MediaOptimization* media_opt);
~VCMEncodedFrameCallback() override;
// Implements EncodedImageCallback.
@@ -76,6 +100,7 @@
rtc::CriticalSection timing_params_lock_;
bool internal_source_;
EncodedImageCallback* const post_encode_callback_;
+ media_optimization::MediaOptimization* const media_opt_;
struct EncodeStartTimeRecord {
EncodeStartTimeRecord(uint32_t timestamp,
@@ -128,8 +153,8 @@
const CodecSpecificInfo* codec_specific,
const std::vector<FrameType>& frame_types);
- void SetEncoderParameters(const VideoBitrateAllocation& target_bitrate,
- uint32_t input_frame_rate);
+ void SetEncoderParameters(const EncoderParameters& params);
+ EncoderParameters GetEncoderParameters() const;
int32_t RequestFrame(const std::vector<FrameType>& frame_types);
bool InternalSource() const;
@@ -142,8 +167,7 @@
VCMEncodedFrameCallback* const vcm_encoded_frame_callback_;
const bool internal_source_;
rtc::CriticalSection params_lock_;
- VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(params_lock_);
- uint32_t input_frame_rate_ RTC_GUARDED_BY(params_lock_);
+ EncoderParameters encoder_params_ RTC_GUARDED_BY(params_lock_);
size_t streams_or_svc_num_ RTC_GUARDED_BY(race_checker_);
VideoCodecType codec_type_ RTC_GUARDED_BY(race_checker_);
};
diff --git a/modules/video_coding/generic_encoder_unittest.cc b/modules/video_coding/generic_encoder_unittest.cc
index 9491b5a..4d5df5b 100644
--- a/modules/video_coding/generic_encoder_unittest.cc
+++ b/modules/video_coding/generic_encoder_unittest.cc
@@ -76,7 +76,7 @@
const int num_streams,
const int num_frames) {
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink);
+ VCMEncodedFrameCallback callback(&sink, nullptr);
const size_t kFramerate = 30;
callback.SetTimingFramesThresholds(
{delay_ms, kDefaultOutlierFrameSizePercent});
@@ -192,7 +192,7 @@
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink);
+ VCMEncodedFrameCallback callback(&sink, nullptr);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
callback.SetTimingFramesThresholds(thresholds);
@@ -223,7 +223,7 @@
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink);
+ VCMEncodedFrameCallback callback(&sink, nullptr);
callback.SetInternalSource(true);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
@@ -258,7 +258,7 @@
const int64_t kTimestampMs4 = 47721870;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink);
+ VCMEncodedFrameCallback callback(&sink, nullptr);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs1;
@@ -293,7 +293,7 @@
const int64_t kTimestampMs = 123456;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink);
+ VCMEncodedFrameCallback callback(&sink, nullptr);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs; // Incorrect timesetamp.
diff --git a/modules/video_coding/media_optimization.cc b/modules/video_coding/media_optimization.cc
new file mode 100644
index 0000000..f74e5eb
--- /dev/null
+++ b/modules/video_coding/media_optimization.cc
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "modules/video_coding/media_optimization.h"
+
+#include <string.h>
+#include <algorithm>
+#include <limits>
+
+#include "modules/video_coding/utility/frame_dropper.h"
+#include "system_wrappers/include/clock.h"
+
+namespace webrtc {
+namespace media_optimization {
+namespace {
+const int64_t kFrameHistoryWinMs = 2000;
+} // namespace
+
+MediaOptimization::MediaOptimization(Clock* clock)
+ : clock_(clock),
+ max_bit_rate_(0),
+ max_frame_rate_(0),
+ frame_dropper_(),
+ incoming_frame_rate_(0) {
+ memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
+ frame_dropper_.SetRates(0, 0);
+}
+
+MediaOptimization::~MediaOptimization() = default;
+
+void MediaOptimization::SetEncodingData(int32_t max_bit_rate,
+ uint32_t target_bitrate,
+ uint32_t max_frame_rate) {
+ rtc::CritScope lock(&crit_sect_);
+ // Everything codec specific should be reset here since the codec has changed.
+ max_bit_rate_ = max_bit_rate;
+ max_frame_rate_ = static_cast<float>(max_frame_rate);
+ float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
+ frame_dropper_.Reset();
+ frame_dropper_.SetRates(target_bitrate_kbps, max_frame_rate_);
+}
+
+uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate) {
+ rtc::CritScope lock(&crit_sect_);
+
+ // Cap target video bitrate to codec maximum.
+ int video_target_bitrate = target_bitrate;
+ if (max_bit_rate_ > 0 && video_target_bitrate > max_bit_rate_) {
+ video_target_bitrate = max_bit_rate_;
+ }
+ float target_video_bitrate_kbps =
+ static_cast<float>(video_target_bitrate) / 1000.0f;
+
+ float framerate = incoming_frame_rate_;
+ if (framerate == 0.0) {
+ // No framerate estimate available, use configured max framerate instead.
+ framerate = max_frame_rate_;
+ }
+
+ frame_dropper_.SetRates(target_video_bitrate_kbps, framerate);
+
+ return video_target_bitrate;
+}
+
+uint32_t MediaOptimization::InputFrameRate() {
+ rtc::CritScope lock(&crit_sect_);
+ return InputFrameRateInternal();
+}
+
+uint32_t MediaOptimization::InputFrameRateInternal() {
+ ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
+ uint32_t framerate = static_cast<uint32_t>(std::min<float>(
+ std::numeric_limits<uint32_t>::max(), incoming_frame_rate_ + 0.5f));
+ return framerate;
+}
+
+void MediaOptimization::UpdateWithEncodedData(
+ const size_t encoded_image_length,
+ const FrameType encoded_image_frametype) {
+ size_t encoded_length = encoded_image_length;
+ rtc::CritScope lock(&crit_sect_);
+ if (encoded_length > 0) {
+ const bool delta_frame = encoded_image_frametype != kVideoFrameKey;
+ frame_dropper_.Fill(encoded_length, delta_frame);
+ }
+}
+
+void MediaOptimization::EnableFrameDropper(bool enable) {
+ rtc::CritScope lock(&crit_sect_);
+ frame_dropper_.Enable(enable);
+}
+
+bool MediaOptimization::DropFrame() {
+ rtc::CritScope lock(&crit_sect_);
+ UpdateIncomingFrameRate();
+ // Leak appropriate number of bytes.
+ frame_dropper_.Leak(static_cast<uint32_t>(InputFrameRateInternal() + 0.5f));
+ return frame_dropper_.DropFrame();
+}
+
+void MediaOptimization::UpdateIncomingFrameRate() {
+ int64_t now = clock_->TimeInMilliseconds();
+ if (incoming_frame_times_[0] == 0) {
+ // No shifting if this is the first time.
+ } else {
+ // Shift all times one step.
+ for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
+ incoming_frame_times_[i + 1] = incoming_frame_times_[i];
+ }
+ }
+ incoming_frame_times_[0] = now;
+ ProcessIncomingFrameRate(now);
+}
+
+// Allowing VCM to keep track of incoming frame rate.
+void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
+ int32_t num = 0;
+ int32_t nr_of_frames = 0;
+ for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
+ if (incoming_frame_times_[num] <= 0 ||
+ // Don't use data older than 2 s.
+ now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
+ break;
+ } else {
+ nr_of_frames++;
+ }
+ }
+ if (num > 1) {
+ const int64_t diff =
+ incoming_frame_times_[0] - incoming_frame_times_[num - 1];
+ incoming_frame_rate_ = 0.0; // No frame rate estimate available.
+ if (diff > 0) {
+ incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
+ }
+ }
+}
+} // namespace media_optimization
+} // namespace webrtc
diff --git a/modules/video_coding/media_optimization.h b/modules/video_coding/media_optimization.h
new file mode 100644
index 0000000..0b6b82f
--- /dev/null
+++ b/modules/video_coding/media_optimization.h
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
+#define MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
+
+#include <memory>
+
+#include "modules/include/module_common_types.h"
+#include "modules/video_coding/include/video_coding.h"
+#include "modules/video_coding/media_opt_util.h"
+#include "modules/video_coding/utility/frame_dropper.h"
+#include "rtc_base/criticalsection.h"
+
+namespace webrtc {
+
+class Clock;
+
+namespace media_optimization {
+
+class MediaOptimization {
+ public:
+ explicit MediaOptimization(Clock* clock);
+ ~MediaOptimization();
+
+ // Informs media optimization of initial encoding state.
+ // TODO(perkj): Deprecate SetEncodingData once its not used for stats in
+ // VideoStreamEncoder.
+ void SetEncodingData(int32_t max_bit_rate,
+ uint32_t bit_rate,
+ uint32_t max_frame_rate);
+
+ // Sets target rates for the encoder given the channel parameters.
+ // Input: |target bitrate| - the encoder target bitrate in bits/s.
+ uint32_t SetTargetRates(uint32_t target_bitrate);
+
+ void EnableFrameDropper(bool enable);
+ bool DropFrame();
+
+ // Informs Media Optimization of encoded output.
+ // TODO(perkj): Deprecate SetEncodingData once its not used for stats in
+ // VideoStreamEncoder.
+ void UpdateWithEncodedData(const size_t encoded_image_length,
+ const FrameType encoded_image_frametype);
+
+ // InputFrameRate 0 = no frame rate estimate available.
+ uint32_t InputFrameRate();
+
+ private:
+ enum { kFrameCountHistorySize = 90 };
+
+ void UpdateIncomingFrameRate() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
+ void ProcessIncomingFrameRate(int64_t now)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
+ uint32_t InputFrameRateInternal() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
+
+ // Protect all members.
+ rtc::CriticalSection crit_sect_;
+
+ Clock* const clock_ RTC_GUARDED_BY(crit_sect_);
+ int32_t max_bit_rate_ RTC_GUARDED_BY(crit_sect_);
+ float max_frame_rate_ RTC_GUARDED_BY(crit_sect_);
+ FrameDropper frame_dropper_ RTC_GUARDED_BY(crit_sect_);
+ float incoming_frame_rate_ RTC_GUARDED_BY(crit_sect_);
+ int64_t incoming_frame_times_[kFrameCountHistorySize] RTC_GUARDED_BY(
+ crit_sect_);
+};
+} // namespace media_optimization
+} // namespace webrtc
+
+#endif // MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
diff --git a/modules/video_coding/video_coding_impl.h b/modules/video_coding/video_coding_impl.h
index 08b0753..e7fb74c 100644
--- a/modules/video_coding/video_coding_impl.h
+++ b/modules/video_coding/video_coding_impl.h
@@ -24,6 +24,7 @@
#include "modules/video_coding/generic_decoder.h"
#include "modules/video_coding/generic_encoder.h"
#include "modules/video_coding/jitter_buffer.h"
+#include "modules/video_coding/media_optimization.h"
#include "modules/video_coding/receiver.h"
#include "modules/video_coding/timing.h"
#include "rtc_base/onetimeevent.h"
@@ -62,6 +63,7 @@
typedef VideoCodingModule::SenderNackMode SenderNackMode;
VideoSender(Clock* clock, EncodedImageCallback* post_encode_callback);
+
~VideoSender();
// Register the send codec to be used.
@@ -73,27 +75,60 @@
void RegisterExternalEncoder(VideoEncoder* externalEncoder,
bool internalSource);
- // Update the the encoder with new bitrate allocation and framerate.
- int32_t SetChannelParameters(const VideoBitrateAllocation& bitrate_allocation,
- uint32_t framerate_fps);
+ // Update the channel parameters based on new rates and rtt. This will also
+ // cause an immediate call to VideoEncoder::SetRateAllocation().
+ int32_t SetChannelParameters(
+ uint32_t target_bitrate_bps,
+ VideoBitrateAllocator* bitrate_allocator,
+ VideoBitrateAllocationObserver* bitrate_updated_callback);
+
+ // Updates the channel parameters with a new bitrate allocation, but using the
+ // current targit_bitrate, loss rate and rtt. That is, the distribution or
+ // caps may be updated to a change to a new VideoCodec or allocation mode.
+ // The new parameters will be stored as pending EncoderParameters, and the
+ // encoder will only be updated on the next frame.
+ void UpdateChannelParameters(
+ VideoBitrateAllocator* bitrate_allocator,
+ VideoBitrateAllocationObserver* bitrate_updated_callback);
int32_t AddVideoFrame(const VideoFrame& videoFrame,
const CodecSpecificInfo* codecSpecificInfo,
absl::optional<VideoEncoder::EncoderInfo> encoder_info);
int32_t IntraFrameRequest(size_t stream_index);
+ int32_t EnableFrameDropper(bool enable);
private:
+ EncoderParameters UpdateEncoderParameters(
+ const EncoderParameters& params,
+ VideoBitrateAllocator* bitrate_allocator,
+ uint32_t target_bitrate_bps);
+ void SetEncoderParameters(EncoderParameters params, bool has_internal_source)
+ RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_);
+ VideoBitrateAllocation GetAllocation(
+ uint32_t bitrate_bps,
+ uint32_t framerate_fps,
+ VideoBitrateAllocator* bitrate_allocator) const;
+
rtc::CriticalSection encoder_crit_;
VCMGenericEncoder* _encoder;
+ media_optimization::MediaOptimization _mediaOpt;
VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_);
+ EncodedImageCallback* const post_encode_callback_;
VCMEncoderDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_);
+ // If frame dropper is not force disabled, frame dropping might still be
+ // disabled if VideoEncoder::GetEncoderInfo() indicates that the encoder has a
+ // trusted rate controller. This is determined on a per-frame basis, as the
+ // encoder behavior might dynamically change.
+ bool force_disable_frame_dropper_ RTC_GUARDED_BY(encoder_crit_);
+
// Must be accessed on the construction thread of VideoSender.
VideoCodec current_codec_;
rtc::SequencedTaskChecker sequenced_checker_;
rtc::CriticalSection params_crit_;
+ EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_);
bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_);
std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_);
};
diff --git a/modules/video_coding/video_sender.cc b/modules/video_coding/video_sender.cc
index fe959b8..aea26f7 100644
--- a/modules/video_coding/video_sender.cc
+++ b/modules/video_coding/video_sender.cc
@@ -25,6 +25,7 @@
#include "modules/video_coding/include/video_coding_defines.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/internal_defines.h"
+#include "modules/video_coding/media_optimization.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
#include "modules/video_coding/video_coding_impl.h"
#include "rtc_base/checks.h"
@@ -38,11 +39,20 @@
namespace webrtc {
namespace vcm {
+namespace {
+
+constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
+
+} // namespace
+
VideoSender::VideoSender(Clock* clock,
EncodedImageCallback* post_encode_callback)
: _encoder(nullptr),
- _encodedFrameCallback(post_encode_callback),
+ _mediaOpt(clock),
+ _encodedFrameCallback(post_encode_callback, &_mediaOpt),
+ post_encode_callback_(post_encode_callback),
_codecDataBase(&_encodedFrameCallback),
+ force_disable_frame_dropper_(false),
current_codec_(),
encoder_has_internal_source_(false),
next_frame_types_(1, kVideoFrameDelta) {
@@ -83,6 +93,27 @@
// SetSendCodec succeeded, _encoder should be set.
RTC_DCHECK(_encoder);
+ int numLayers;
+ if (sendCodec->codecType == kVideoCodecVP8) {
+ numLayers = sendCodec->VP8().numberOfTemporalLayers;
+ } else if (sendCodec->codecType == kVideoCodecVP9) {
+ numLayers = sendCodec->VP9().numberOfTemporalLayers;
+ } else if (sendCodec->codecType == kVideoCodecGeneric &&
+ sendCodec->numberOfSimulcastStreams > 0) {
+ // This is mainly for unit testing, disabling frame dropping.
+ // TODO(sprang): Add a better way to disable frame dropping.
+ numLayers = sendCodec->simulcastStream[0].numberOfTemporalLayers;
+ } else {
+ numLayers = 1;
+ }
+
+ // Force-disable frame dropper if either:
+ // * We have screensharing with layers.
+ // * "WebRTC-FrameDropper" field trial is "Disabled".
+ force_disable_frame_dropper_ =
+ field_trial::IsDisabled(kFrameDropperFieldTrial) ||
+ (numLayers > 1 && sendCodec->mode == VideoCodecMode::kScreensharing);
+
{
rtc::CritScope cs(¶ms_crit_);
next_frame_types_.clear();
@@ -97,6 +128,9 @@
<< " start bitrate " << sendCodec->startBitrate
<< " max frame rate " << sendCodec->maxFramerate
<< " max payload size " << maxPayloadSize;
+ _mediaOpt.SetEncodingData(sendCodec->maxBitrate * 1000,
+ sendCodec->startBitrate * 1000,
+ sendCodec->maxFramerate);
return VCM_OK;
}
@@ -122,58 +156,149 @@
internalSource);
}
+EncoderParameters VideoSender::UpdateEncoderParameters(
+ const EncoderParameters& params,
+ VideoBitrateAllocator* bitrate_allocator,
+ uint32_t target_bitrate_bps) {
+ uint32_t video_target_rate_bps = _mediaOpt.SetTargetRates(target_bitrate_bps);
+ uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
+ if (input_frame_rate == 0)
+ input_frame_rate = current_codec_.maxFramerate;
+
+ EncoderParameters new_encoder_params = {
+ DataRate::bps(target_bitrate_bps),
+ GetAllocation(video_target_rate_bps, input_frame_rate, bitrate_allocator),
+ input_frame_rate};
+ return new_encoder_params;
+}
+
+VideoBitrateAllocation VideoSender::GetAllocation(
+ uint32_t bitrate_bps,
+ uint32_t framerate_fps,
+ VideoBitrateAllocator* bitrate_allocator) const {
+ VideoBitrateAllocation bitrate_allocation;
+ // Only call allocators if bitrate > 0 (ie, not suspended), otherwise they
+ // might cap the bitrate to the min bitrate configured.
+ if (bitrate_bps > 0) {
+ if (bitrate_allocator) {
+ bitrate_allocation =
+ bitrate_allocator->GetAllocation(bitrate_bps, framerate_fps);
+ } else {
+ DefaultVideoBitrateAllocator default_allocator(current_codec_);
+ bitrate_allocation =
+ default_allocator.GetAllocation(bitrate_bps, framerate_fps);
+ }
+ }
+ return bitrate_allocation;
+}
+
+void VideoSender::UpdateChannelParameters(
+ VideoBitrateAllocator* bitrate_allocator,
+ VideoBitrateAllocationObserver* bitrate_updated_callback) {
+ VideoBitrateAllocation target_rate;
+ {
+ rtc::CritScope cs(¶ms_crit_);
+ encoder_params_ =
+ UpdateEncoderParameters(encoder_params_, bitrate_allocator,
+ encoder_params_.total_bitrate.bps());
+ target_rate = encoder_params_.target_bitrate;
+ }
+ if (bitrate_updated_callback && target_rate.get_sum_bps() > 0) {
+ bitrate_updated_callback->OnBitrateAllocationUpdated(target_rate);
+ }
+}
+
int32_t VideoSender::SetChannelParameters(
- const VideoBitrateAllocation& bitrate_allocation,
- uint32_t framerate_fps) {
+ uint32_t target_bitrate_bps,
+ VideoBitrateAllocator* bitrate_allocator,
+ VideoBitrateAllocationObserver* bitrate_updated_callback) {
+ EncoderParameters encoder_params;
+ encoder_params = UpdateEncoderParameters(encoder_params, bitrate_allocator,
+ target_bitrate_bps);
+ if (bitrate_updated_callback && target_bitrate_bps > 0) {
+ bitrate_updated_callback->OnBitrateAllocationUpdated(
+ encoder_params.target_bitrate);
+ }
+
bool encoder_has_internal_source;
{
rtc::CritScope cs(¶ms_crit_);
+ encoder_params_ = encoder_params;
encoder_has_internal_source = encoder_has_internal_source_;
}
- {
+ // For encoders with internal sources, we need to tell the encoder directly,
+ // instead of waiting for an AddVideoFrame that will never come (internal
+ // source encoders don't get input frames).
+ if (encoder_has_internal_source) {
rtc::CritScope cs(&encoder_crit_);
if (_encoder) {
- // |target_bitrate == 0 | means that the network is down or the send pacer
- // is full. We currently only report this if the encoder has an internal
- // source. If the encoder does not have an internal source, higher levels
- // are expected to not call AddVideoFrame. We do this since its unclear
- // how current encoder implementations behave when given a zero target
- // bitrate.
- // TODO(perkj): Make sure all known encoder implementations handle zero
- // target bitrate and remove this check.
- if (!encoder_has_internal_source &&
- bitrate_allocation.get_sum_bps() == 0) {
- return VCM_OK;
- }
-
- if (framerate_fps == 0) {
- // No frame rate estimate available, use default.
- framerate_fps = current_codec_.maxFramerate;
- }
- if (_encoder != nullptr)
- _encoder->SetEncoderParameters(bitrate_allocation, framerate_fps);
+ SetEncoderParameters(encoder_params, encoder_has_internal_source);
}
}
return VCM_OK;
}
+void VideoSender::SetEncoderParameters(EncoderParameters params,
+ bool has_internal_source) {
+ // |target_bitrate == 0 | means that the network is down or the send pacer is
+ // full. We currently only report this if the encoder has an internal source.
+ // If the encoder does not have an internal source, higher levels are expected
+ // to not call AddVideoFrame. We do this since its unclear how current
+ // encoder implementations behave when given a zero target bitrate.
+ // TODO(perkj): Make sure all known encoder implementations handle zero
+ // target bitrate and remove this check.
+ if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0)
+ return;
+
+ if (params.input_frame_rate == 0) {
+ // No frame rate estimate available, use default.
+ params.input_frame_rate = current_codec_.maxFramerate;
+ }
+ if (_encoder != nullptr)
+ _encoder->SetEncoderParameters(params);
+}
+
// Add one raw video frame to the encoder, blocking.
int32_t VideoSender::AddVideoFrame(
const VideoFrame& videoFrame,
const CodecSpecificInfo* codecSpecificInfo,
absl::optional<VideoEncoder::EncoderInfo> encoder_info) {
+ EncoderParameters encoder_params;
std::vector<FrameType> next_frame_types;
bool encoder_has_internal_source = false;
{
rtc::CritScope lock(¶ms_crit_);
+ encoder_params = encoder_params_;
next_frame_types = next_frame_types_;
encoder_has_internal_source = encoder_has_internal_source_;
}
rtc::CritScope lock(&encoder_crit_);
if (_encoder == nullptr)
return VCM_UNINITIALIZED;
+ SetEncoderParameters(encoder_params, encoder_has_internal_source);
+ if (!encoder_info) {
+ encoder_info = _encoder->GetEncoderInfo();
+ }
+
+ // Frame dropping is enabled iff frame dropping has been requested, and
+ // frame dropping is not force-disabled, and rate controller is not trusted.
+ const bool frame_dropping_enabled =
+ !force_disable_frame_dropper_ &&
+ !encoder_info->has_trusted_rate_controller;
+ _mediaOpt.EnableFrameDropper(frame_dropping_enabled);
+
+ if (_mediaOpt.DropFrame()) {
+ RTC_LOG(LS_VERBOSE) << "Drop Frame: "
+ << "target bitrate "
+ << encoder_params.target_bitrate.get_sum_bps()
+ << ", input frame rate "
+ << encoder_params.input_frame_rate;
+ post_encode_callback_->OnDroppedFrame(
+ EncodedImageCallback::DropReason::kDroppedByMediaOptimizations);
+ return VCM_OK;
+ }
// TODO(pbos): Make sure setting send codec is synchronized with video
// processing so frame size always matches.
if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
diff --git a/modules/video_coding/video_sender_unittest.cc b/modules/video_coding/video_sender_unittest.cc
index a9c9e45..986bd3f 100644
--- a/modules/video_coding/video_sender_unittest.cc
+++ b/modules/video_coding/video_sender_unittest.cc
@@ -319,14 +319,14 @@
SetRateAllocation(new_rate_allocation, settings_.maxFramerate))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
+ sender_->SetChannelParameters(new_bitrate_kbps * 1000, rate_allocator_.get(),
+ nullptr);
AddFrame();
clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
// Expect no call to encoder_.SetRates if the new bitrate is zero.
EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0);
- sender_->SetChannelParameters(VideoBitrateAllocation(),
- settings_.maxFramerate);
+ sender_->SetChannelParameters(0, rate_allocator_.get(), nullptr);
AddFrame();
}
@@ -363,7 +363,8 @@
EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
+ sender_->SetChannelParameters(new_bitrate_kbps * 1000, rate_allocator_.get(),
+ nullptr);
}
TEST_F(TestVideoSenderWithMockEncoder,
@@ -374,9 +375,8 @@
// Expect initial call to SetChannelParameters. Rates are initialized through
// InitEncode and expects no additional call before the framerate (or bitrate)
// updates.
- sender_->SetChannelParameters(
- rate_allocator_->GetAllocation(settings_.startBitrate * 1000, kInputFps),
- kInputFps);
+ sender_->SetChannelParameters(settings_.startBitrate * 1000,
+ rate_allocator_.get(), nullptr);
while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
AddFrame();
clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
@@ -390,7 +390,8 @@
EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_rate_allocation, kInputFps);
+ sender_->SetChannelParameters(new_bitrate_bps, rate_allocator_.get(),
+ nullptr);
AddFrame();
}
@@ -438,13 +439,12 @@
AddFrame();
// SetChannelParameters needs to be called frequently to propagate
// framerate from the media optimization into the encoder.
- const VideoBitrateAllocation bitrate_allocation =
- rate_allocator_->GetAllocation(available_bitrate_kbps_ * 1000,
- static_cast<uint32_t>(framerate));
+ // Note: SetChannelParameters fails if less than 2 frames are in the
+ // buffer since it will fail to calculate the framerate.
if (i != 0) {
- EXPECT_EQ(VCM_OK,
- sender_->SetChannelParameters(
- bitrate_allocation, static_cast<uint32_t>(framerate)));
+ EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
+ available_bitrate_kbps_ * 1000,
+ rate_allocator_.get(), nullptr));
}
}
}