Reland "Refactor and remove media_optimization::MediaOptimization."
This reverts commit 6613f8e98ab3654ade7e8f5352d8d6711b157499.
Reason for revert: This change seemed innocent after all, so undoing speculative revert.
Original change's description:
> Revert "Refactor and remove media_optimization::MediaOptimization."
>
> This reverts commit 07276e4f89a93b1479d7aeefa53b4fc32daf001b.
>
> Reason for revert: Speculative revert due to downstream crashes.
>
> Original change's description:
> > Refactor and remove media_optimization::MediaOptimization.
> >
> > This CL removes MediaOptmization and folds some of its functionality
> > into VideoStreamEncoder.
> >
> > The FPS tracking is now handled by a RateStatistics instance. Frame
> > dropping is still handled by FrameDropper. Both of these now live
> > directly in VideoStreamEncoder.
> > There is no intended change in behavior from this CL, but due to a new
> > way of measuring frame rate, some minor perf changes can be expected.
> >
> > A small change in behavior is that OnBitrateUpdated is now called
> > directly rather than on the next frame. Since both encoding frame and
> > setting rate allocations happen on the encoder worker thread, there's
> > really no reason to cache bitrates and wait until the next frame.
> > An edge case though is that if a new bitrate is set before the first
> > frame, we must remember that bitrate and then apply it after the video
> > bitrate allocator has been first created.
> >
> > In addition to existing unit tests, manual tests have been used to
> > confirm that frame dropping works as expected with misbehaving encoders.
> >
> > Bug: webrtc:10164
> > Change-Id: I7ee9c8d3c4f2bcf23c8c420310b05a4d35d94744
> > Reviewed-on: https://webrtc-review.googlesource.com/c/115620
> > Commit-Queue: Erik Språng <sprang@webrtc.org>
> > Reviewed-by: Niels Moller <nisse@webrtc.org>
> > Cr-Commit-Position: refs/heads/master@{#26147}
>
> TBR=nisse@webrtc.org,sprang@webrtc.org
>
> # Not skipping CQ checks because original CL landed > 1 day ago.
>
> Bug: webrtc:10164
> Change-Id: Ie0dae19dd012bc09e793c9661a45823fd760c20c
> Reviewed-on: https://webrtc-review.googlesource.com/c/116780
> Reviewed-by: Niels Moller <nisse@webrtc.org>
> Reviewed-by: Erik Språng <sprang@webrtc.org>
> Commit-Queue: Niels Moller <nisse@webrtc.org>
> Cr-Commit-Position: refs/heads/master@{#26191}
TBR=nisse@webrtc.org,sprang@webrtc.org
Change-Id: Ieda1fad301de002460bb0bf5a75267ea065176a8
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: webrtc:10164
Reviewed-on: https://webrtc-review.googlesource.com/c/116960
Reviewed-by: Niels Moller <nisse@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Niels Moller <nisse@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26213}diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 6eec2f4..188b624 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -133,8 +133,6 @@
"jitter_estimator.h",
"media_opt_util.cc",
"media_opt_util.h",
- "media_optimization.cc",
- "media_optimization.h",
"nack_fec_tables.h",
"packet_buffer.cc",
"packet_buffer.h",
diff --git a/modules/video_coding/generic_encoder.cc b/modules/video_coding/generic_encoder.cc
index 9dd07bc..2d992ef 100644
--- a/modules/video_coding/generic_encoder.cc
+++ b/modules/video_coding/generic_encoder.cc
@@ -22,7 +22,6 @@
#include "api/video/video_timing.h"
#include "modules/include/module_common_types_public.h"
#include "modules/video_coding/include/video_coding_defines.h"
-#include "modules/video_coding/media_optimization.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/alr_experiment.h"
#include "rtc_base/logging.h"
@@ -46,6 +45,7 @@
: encoder_(encoder),
vcm_encoded_frame_callback_(encoded_frame_callback),
internal_source_(internal_source),
+ input_frame_rate_(0),
streams_or_svc_num_(0),
codec_type_(VideoCodecType::kVideoCodecGeneric) {}
@@ -103,38 +103,34 @@
return encoder_->Encode(frame, codec_specific, &frame_types);
}
-void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) {
+void VCMGenericEncoder::SetEncoderParameters(
+ const VideoBitrateAllocation& target_bitrate,
+ uint32_t input_frame_rate) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
bool rates_have_changed;
{
rtc::CritScope lock(¶ms_lock_);
- rates_have_changed =
- params.target_bitrate != encoder_params_.target_bitrate ||
- params.input_frame_rate != encoder_params_.input_frame_rate;
- encoder_params_ = params;
+ rates_have_changed = target_bitrate != bitrate_allocation_ ||
+ input_frame_rate != input_frame_rate_;
+ bitrate_allocation_ = target_bitrate;
+ input_frame_rate_ = input_frame_rate;
}
if (rates_have_changed) {
- int res = encoder_->SetRateAllocation(params.target_bitrate,
- params.input_frame_rate);
+ int res = encoder_->SetRateAllocation(target_bitrate, input_frame_rate);
if (res != 0) {
RTC_LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
- << params.target_bitrate.get_sum_bps()
- << ", framerate = " << params.input_frame_rate
+ << target_bitrate.get_sum_bps()
+ << ", framerate = " << input_frame_rate
<< "): " << res;
}
- vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate);
+ vcm_encoded_frame_callback_->OnFrameRateChanged(input_frame_rate);
for (size_t i = 0; i < streams_or_svc_num_; ++i) {
vcm_encoded_frame_callback_->OnTargetBitrateChanged(
- params.target_bitrate.GetSpatialLayerSum(i) / 8, i);
+ target_bitrate.GetSpatialLayerSum(i) / 8, i);
}
}
}
-EncoderParameters VCMGenericEncoder::GetEncoderParameters() const {
- rtc::CritScope lock(¶ms_lock_);
- return encoder_params_;
-}
-
int32_t VCMGenericEncoder::RequestFrame(
const std::vector<FrameType>& frame_types) {
RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
@@ -164,11 +160,9 @@
}
VCMEncodedFrameCallback::VCMEncodedFrameCallback(
- EncodedImageCallback* post_encode_callback,
- media_optimization::MediaOptimization* media_opt)
+ EncodedImageCallback* post_encode_callback)
: internal_source_(false),
post_encode_callback_(post_encode_callback),
- media_opt_(media_opt),
framerate_(1),
last_timing_frame_time_ms_(-1),
timing_frames_thresholds_({-1, 0}),
@@ -406,20 +400,8 @@
RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
&image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
- Result result = post_encode_callback_->OnEncodedImage(
- image_copy, codec_specific, fragmentation_header);
- if (result.error != Result::OK)
- return result;
-
- if (media_opt_) {
- media_opt_->UpdateWithEncodedData(image_copy._length,
- image_copy._frameType);
- if (internal_source_) {
- // Signal to encoder to drop next frame.
- result.drop_next_frame = media_opt_->DropFrame();
- }
- }
- return result;
+ return post_encode_callback_->OnEncodedImage(image_copy, codec_specific,
+ fragmentation_header);
}
} // namespace webrtc
diff --git a/modules/video_coding/generic_encoder.h b/modules/video_coding/generic_encoder.h
index 1f4ade2..463f98a 100644
--- a/modules/video_coding/generic_encoder.h
+++ b/modules/video_coding/generic_encoder.h
@@ -23,33 +23,9 @@
namespace webrtc {
-namespace media_optimization {
-class MediaOptimization;
-} // namespace media_optimization
-
-struct EncoderParameters {
- EncoderParameters() : total_bitrate(DataRate::Zero()), input_frame_rate(0) {}
- EncoderParameters(DataRate total_bitrate,
- const VideoBitrateAllocation& allocation,
- uint32_t framerate)
- : total_bitrate(total_bitrate),
- target_bitrate(allocation),
- input_frame_rate(framerate) {}
-
- // Total bitrate allocated for this encoder.
- DataRate total_bitrate;
- // The bitrate allocation, across spatial and/or temporal layers. Note that
- // the sum of these might be less than |total_bitrate| if the allocator has
- // capped the bitrate for some configuration.
- VideoBitrateAllocation target_bitrate;
- // The input frame rate to the encoder in fps, measured in the video sender.
- uint32_t input_frame_rate;
-};
-
class VCMEncodedFrameCallback : public EncodedImageCallback {
public:
- VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback,
- media_optimization::MediaOptimization* media_opt);
+ explicit VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback);
~VCMEncodedFrameCallback() override;
// Implements EncodedImageCallback.
@@ -100,7 +76,6 @@
rtc::CriticalSection timing_params_lock_;
bool internal_source_;
EncodedImageCallback* const post_encode_callback_;
- media_optimization::MediaOptimization* const media_opt_;
struct EncodeStartTimeRecord {
EncodeStartTimeRecord(uint32_t timestamp,
@@ -153,8 +128,8 @@
const CodecSpecificInfo* codec_specific,
const std::vector<FrameType>& frame_types);
- void SetEncoderParameters(const EncoderParameters& params);
- EncoderParameters GetEncoderParameters() const;
+ void SetEncoderParameters(const VideoBitrateAllocation& target_bitrate,
+ uint32_t input_frame_rate);
int32_t RequestFrame(const std::vector<FrameType>& frame_types);
bool InternalSource() const;
@@ -167,7 +142,8 @@
VCMEncodedFrameCallback* const vcm_encoded_frame_callback_;
const bool internal_source_;
rtc::CriticalSection params_lock_;
- EncoderParameters encoder_params_ RTC_GUARDED_BY(params_lock_);
+ VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(params_lock_);
+ uint32_t input_frame_rate_ RTC_GUARDED_BY(params_lock_);
size_t streams_or_svc_num_ RTC_GUARDED_BY(race_checker_);
VideoCodecType codec_type_ RTC_GUARDED_BY(race_checker_);
};
diff --git a/modules/video_coding/generic_encoder_unittest.cc b/modules/video_coding/generic_encoder_unittest.cc
index 4d5df5b..9491b5a 100644
--- a/modules/video_coding/generic_encoder_unittest.cc
+++ b/modules/video_coding/generic_encoder_unittest.cc
@@ -76,7 +76,7 @@
const int num_streams,
const int num_frames) {
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
const size_t kFramerate = 30;
callback.SetTimingFramesThresholds(
{delay_ms, kDefaultOutlierFrameSizePercent});
@@ -192,7 +192,7 @@
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
callback.SetTimingFramesThresholds(thresholds);
@@ -223,7 +223,7 @@
image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
callback.SetInternalSource(true);
VideoCodec::TimingFrameTriggerThresholds thresholds;
thresholds.delay_ms = 1; // Make all frames timing frames.
@@ -258,7 +258,7 @@
const int64_t kTimestampMs4 = 47721870;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs1;
@@ -293,7 +293,7 @@
const int64_t kTimestampMs = 123456;
codec_specific.codecType = kVideoCodecGeneric;
FakeEncodedImageCallback sink;
- VCMEncodedFrameCallback callback(&sink, nullptr);
+ VCMEncodedFrameCallback callback(&sink);
// Any non-zero bitrate needed to be set before the first frame.
callback.OnTargetBitrateChanged(500, 0);
image.capture_time_ms_ = kTimestampMs; // Incorrect timesetamp.
diff --git a/modules/video_coding/media_optimization.cc b/modules/video_coding/media_optimization.cc
deleted file mode 100644
index f74e5eb..0000000
--- a/modules/video_coding/media_optimization.cc
+++ /dev/null
@@ -1,145 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#include "modules/video_coding/media_optimization.h"
-
-#include <string.h>
-#include <algorithm>
-#include <limits>
-
-#include "modules/video_coding/utility/frame_dropper.h"
-#include "system_wrappers/include/clock.h"
-
-namespace webrtc {
-namespace media_optimization {
-namespace {
-const int64_t kFrameHistoryWinMs = 2000;
-} // namespace
-
-MediaOptimization::MediaOptimization(Clock* clock)
- : clock_(clock),
- max_bit_rate_(0),
- max_frame_rate_(0),
- frame_dropper_(),
- incoming_frame_rate_(0) {
- memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_));
- frame_dropper_.SetRates(0, 0);
-}
-
-MediaOptimization::~MediaOptimization() = default;
-
-void MediaOptimization::SetEncodingData(int32_t max_bit_rate,
- uint32_t target_bitrate,
- uint32_t max_frame_rate) {
- rtc::CritScope lock(&crit_sect_);
- // Everything codec specific should be reset here since the codec has changed.
- max_bit_rate_ = max_bit_rate;
- max_frame_rate_ = static_cast<float>(max_frame_rate);
- float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f;
- frame_dropper_.Reset();
- frame_dropper_.SetRates(target_bitrate_kbps, max_frame_rate_);
-}
-
-uint32_t MediaOptimization::SetTargetRates(uint32_t target_bitrate) {
- rtc::CritScope lock(&crit_sect_);
-
- // Cap target video bitrate to codec maximum.
- int video_target_bitrate = target_bitrate;
- if (max_bit_rate_ > 0 && video_target_bitrate > max_bit_rate_) {
- video_target_bitrate = max_bit_rate_;
- }
- float target_video_bitrate_kbps =
- static_cast<float>(video_target_bitrate) / 1000.0f;
-
- float framerate = incoming_frame_rate_;
- if (framerate == 0.0) {
- // No framerate estimate available, use configured max framerate instead.
- framerate = max_frame_rate_;
- }
-
- frame_dropper_.SetRates(target_video_bitrate_kbps, framerate);
-
- return video_target_bitrate;
-}
-
-uint32_t MediaOptimization::InputFrameRate() {
- rtc::CritScope lock(&crit_sect_);
- return InputFrameRateInternal();
-}
-
-uint32_t MediaOptimization::InputFrameRateInternal() {
- ProcessIncomingFrameRate(clock_->TimeInMilliseconds());
- uint32_t framerate = static_cast<uint32_t>(std::min<float>(
- std::numeric_limits<uint32_t>::max(), incoming_frame_rate_ + 0.5f));
- return framerate;
-}
-
-void MediaOptimization::UpdateWithEncodedData(
- const size_t encoded_image_length,
- const FrameType encoded_image_frametype) {
- size_t encoded_length = encoded_image_length;
- rtc::CritScope lock(&crit_sect_);
- if (encoded_length > 0) {
- const bool delta_frame = encoded_image_frametype != kVideoFrameKey;
- frame_dropper_.Fill(encoded_length, delta_frame);
- }
-}
-
-void MediaOptimization::EnableFrameDropper(bool enable) {
- rtc::CritScope lock(&crit_sect_);
- frame_dropper_.Enable(enable);
-}
-
-bool MediaOptimization::DropFrame() {
- rtc::CritScope lock(&crit_sect_);
- UpdateIncomingFrameRate();
- // Leak appropriate number of bytes.
- frame_dropper_.Leak(static_cast<uint32_t>(InputFrameRateInternal() + 0.5f));
- return frame_dropper_.DropFrame();
-}
-
-void MediaOptimization::UpdateIncomingFrameRate() {
- int64_t now = clock_->TimeInMilliseconds();
- if (incoming_frame_times_[0] == 0) {
- // No shifting if this is the first time.
- } else {
- // Shift all times one step.
- for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) {
- incoming_frame_times_[i + 1] = incoming_frame_times_[i];
- }
- }
- incoming_frame_times_[0] = now;
- ProcessIncomingFrameRate(now);
-}
-
-// Allowing VCM to keep track of incoming frame rate.
-void MediaOptimization::ProcessIncomingFrameRate(int64_t now) {
- int32_t num = 0;
- int32_t nr_of_frames = 0;
- for (num = 1; num < (kFrameCountHistorySize - 1); ++num) {
- if (incoming_frame_times_[num] <= 0 ||
- // Don't use data older than 2 s.
- now - incoming_frame_times_[num] > kFrameHistoryWinMs) {
- break;
- } else {
- nr_of_frames++;
- }
- }
- if (num > 1) {
- const int64_t diff =
- incoming_frame_times_[0] - incoming_frame_times_[num - 1];
- incoming_frame_rate_ = 0.0; // No frame rate estimate available.
- if (diff > 0) {
- incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff);
- }
- }
-}
-} // namespace media_optimization
-} // namespace webrtc
diff --git a/modules/video_coding/media_optimization.h b/modules/video_coding/media_optimization.h
deleted file mode 100644
index 0b6b82f..0000000
--- a/modules/video_coding/media_optimization.h
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
- *
- * Use of this source code is governed by a BSD-style license
- * that can be found in the LICENSE file in the root of the source
- * tree. An additional intellectual property rights grant can be found
- * in the file PATENTS. All contributing project authors may
- * be found in the AUTHORS file in the root of the source tree.
- */
-
-#ifndef MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
-#define MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
-
-#include <memory>
-
-#include "modules/include/module_common_types.h"
-#include "modules/video_coding/include/video_coding.h"
-#include "modules/video_coding/media_opt_util.h"
-#include "modules/video_coding/utility/frame_dropper.h"
-#include "rtc_base/criticalsection.h"
-
-namespace webrtc {
-
-class Clock;
-
-namespace media_optimization {
-
-class MediaOptimization {
- public:
- explicit MediaOptimization(Clock* clock);
- ~MediaOptimization();
-
- // Informs media optimization of initial encoding state.
- // TODO(perkj): Deprecate SetEncodingData once its not used for stats in
- // VideoStreamEncoder.
- void SetEncodingData(int32_t max_bit_rate,
- uint32_t bit_rate,
- uint32_t max_frame_rate);
-
- // Sets target rates for the encoder given the channel parameters.
- // Input: |target bitrate| - the encoder target bitrate in bits/s.
- uint32_t SetTargetRates(uint32_t target_bitrate);
-
- void EnableFrameDropper(bool enable);
- bool DropFrame();
-
- // Informs Media Optimization of encoded output.
- // TODO(perkj): Deprecate SetEncodingData once its not used for stats in
- // VideoStreamEncoder.
- void UpdateWithEncodedData(const size_t encoded_image_length,
- const FrameType encoded_image_frametype);
-
- // InputFrameRate 0 = no frame rate estimate available.
- uint32_t InputFrameRate();
-
- private:
- enum { kFrameCountHistorySize = 90 };
-
- void UpdateIncomingFrameRate() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
- void ProcessIncomingFrameRate(int64_t now)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
- uint32_t InputFrameRateInternal() RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_sect_);
-
- // Protect all members.
- rtc::CriticalSection crit_sect_;
-
- Clock* const clock_ RTC_GUARDED_BY(crit_sect_);
- int32_t max_bit_rate_ RTC_GUARDED_BY(crit_sect_);
- float max_frame_rate_ RTC_GUARDED_BY(crit_sect_);
- FrameDropper frame_dropper_ RTC_GUARDED_BY(crit_sect_);
- float incoming_frame_rate_ RTC_GUARDED_BY(crit_sect_);
- int64_t incoming_frame_times_[kFrameCountHistorySize] RTC_GUARDED_BY(
- crit_sect_);
-};
-} // namespace media_optimization
-} // namespace webrtc
-
-#endif // MODULES_VIDEO_CODING_MEDIA_OPTIMIZATION_H_
diff --git a/modules/video_coding/video_coding_impl.h b/modules/video_coding/video_coding_impl.h
index e7fb74c..08b0753 100644
--- a/modules/video_coding/video_coding_impl.h
+++ b/modules/video_coding/video_coding_impl.h
@@ -24,7 +24,6 @@
#include "modules/video_coding/generic_decoder.h"
#include "modules/video_coding/generic_encoder.h"
#include "modules/video_coding/jitter_buffer.h"
-#include "modules/video_coding/media_optimization.h"
#include "modules/video_coding/receiver.h"
#include "modules/video_coding/timing.h"
#include "rtc_base/onetimeevent.h"
@@ -63,7 +62,6 @@
typedef VideoCodingModule::SenderNackMode SenderNackMode;
VideoSender(Clock* clock, EncodedImageCallback* post_encode_callback);
-
~VideoSender();
// Register the send codec to be used.
@@ -75,60 +73,27 @@
void RegisterExternalEncoder(VideoEncoder* externalEncoder,
bool internalSource);
- // Update the channel parameters based on new rates and rtt. This will also
- // cause an immediate call to VideoEncoder::SetRateAllocation().
- int32_t SetChannelParameters(
- uint32_t target_bitrate_bps,
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback);
-
- // Updates the channel parameters with a new bitrate allocation, but using the
- // current targit_bitrate, loss rate and rtt. That is, the distribution or
- // caps may be updated to a change to a new VideoCodec or allocation mode.
- // The new parameters will be stored as pending EncoderParameters, and the
- // encoder will only be updated on the next frame.
- void UpdateChannelParameters(
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback);
+ // Update the the encoder with new bitrate allocation and framerate.
+ int32_t SetChannelParameters(const VideoBitrateAllocation& bitrate_allocation,
+ uint32_t framerate_fps);
int32_t AddVideoFrame(const VideoFrame& videoFrame,
const CodecSpecificInfo* codecSpecificInfo,
absl::optional<VideoEncoder::EncoderInfo> encoder_info);
int32_t IntraFrameRequest(size_t stream_index);
- int32_t EnableFrameDropper(bool enable);
private:
- EncoderParameters UpdateEncoderParameters(
- const EncoderParameters& params,
- VideoBitrateAllocator* bitrate_allocator,
- uint32_t target_bitrate_bps);
- void SetEncoderParameters(EncoderParameters params, bool has_internal_source)
- RTC_EXCLUSIVE_LOCKS_REQUIRED(encoder_crit_);
- VideoBitrateAllocation GetAllocation(
- uint32_t bitrate_bps,
- uint32_t framerate_fps,
- VideoBitrateAllocator* bitrate_allocator) const;
-
rtc::CriticalSection encoder_crit_;
VCMGenericEncoder* _encoder;
- media_optimization::MediaOptimization _mediaOpt;
VCMEncodedFrameCallback _encodedFrameCallback RTC_GUARDED_BY(encoder_crit_);
- EncodedImageCallback* const post_encode_callback_;
VCMEncoderDataBase _codecDataBase RTC_GUARDED_BY(encoder_crit_);
- // If frame dropper is not force disabled, frame dropping might still be
- // disabled if VideoEncoder::GetEncoderInfo() indicates that the encoder has a
- // trusted rate controller. This is determined on a per-frame basis, as the
- // encoder behavior might dynamically change.
- bool force_disable_frame_dropper_ RTC_GUARDED_BY(encoder_crit_);
-
// Must be accessed on the construction thread of VideoSender.
VideoCodec current_codec_;
rtc::SequencedTaskChecker sequenced_checker_;
rtc::CriticalSection params_crit_;
- EncoderParameters encoder_params_ RTC_GUARDED_BY(params_crit_);
bool encoder_has_internal_source_ RTC_GUARDED_BY(params_crit_);
std::vector<FrameType> next_frame_types_ RTC_GUARDED_BY(params_crit_);
};
diff --git a/modules/video_coding/video_sender.cc b/modules/video_coding/video_sender.cc
index aea26f7..fe959b8 100644
--- a/modules/video_coding/video_sender.cc
+++ b/modules/video_coding/video_sender.cc
@@ -25,7 +25,6 @@
#include "modules/video_coding/include/video_coding_defines.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "modules/video_coding/internal_defines.h"
-#include "modules/video_coding/media_optimization.h"
#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
#include "modules/video_coding/video_coding_impl.h"
#include "rtc_base/checks.h"
@@ -39,20 +38,11 @@
namespace webrtc {
namespace vcm {
-namespace {
-
-constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
-
-} // namespace
-
VideoSender::VideoSender(Clock* clock,
EncodedImageCallback* post_encode_callback)
: _encoder(nullptr),
- _mediaOpt(clock),
- _encodedFrameCallback(post_encode_callback, &_mediaOpt),
- post_encode_callback_(post_encode_callback),
+ _encodedFrameCallback(post_encode_callback),
_codecDataBase(&_encodedFrameCallback),
- force_disable_frame_dropper_(false),
current_codec_(),
encoder_has_internal_source_(false),
next_frame_types_(1, kVideoFrameDelta) {
@@ -93,27 +83,6 @@
// SetSendCodec succeeded, _encoder should be set.
RTC_DCHECK(_encoder);
- int numLayers;
- if (sendCodec->codecType == kVideoCodecVP8) {
- numLayers = sendCodec->VP8().numberOfTemporalLayers;
- } else if (sendCodec->codecType == kVideoCodecVP9) {
- numLayers = sendCodec->VP9().numberOfTemporalLayers;
- } else if (sendCodec->codecType == kVideoCodecGeneric &&
- sendCodec->numberOfSimulcastStreams > 0) {
- // This is mainly for unit testing, disabling frame dropping.
- // TODO(sprang): Add a better way to disable frame dropping.
- numLayers = sendCodec->simulcastStream[0].numberOfTemporalLayers;
- } else {
- numLayers = 1;
- }
-
- // Force-disable frame dropper if either:
- // * We have screensharing with layers.
- // * "WebRTC-FrameDropper" field trial is "Disabled".
- force_disable_frame_dropper_ =
- field_trial::IsDisabled(kFrameDropperFieldTrial) ||
- (numLayers > 1 && sendCodec->mode == VideoCodecMode::kScreensharing);
-
{
rtc::CritScope cs(¶ms_crit_);
next_frame_types_.clear();
@@ -128,9 +97,6 @@
<< " start bitrate " << sendCodec->startBitrate
<< " max frame rate " << sendCodec->maxFramerate
<< " max payload size " << maxPayloadSize;
- _mediaOpt.SetEncodingData(sendCodec->maxBitrate * 1000,
- sendCodec->startBitrate * 1000,
- sendCodec->maxFramerate);
return VCM_OK;
}
@@ -156,149 +122,58 @@
internalSource);
}
-EncoderParameters VideoSender::UpdateEncoderParameters(
- const EncoderParameters& params,
- VideoBitrateAllocator* bitrate_allocator,
- uint32_t target_bitrate_bps) {
- uint32_t video_target_rate_bps = _mediaOpt.SetTargetRates(target_bitrate_bps);
- uint32_t input_frame_rate = _mediaOpt.InputFrameRate();
- if (input_frame_rate == 0)
- input_frame_rate = current_codec_.maxFramerate;
-
- EncoderParameters new_encoder_params = {
- DataRate::bps(target_bitrate_bps),
- GetAllocation(video_target_rate_bps, input_frame_rate, bitrate_allocator),
- input_frame_rate};
- return new_encoder_params;
-}
-
-VideoBitrateAllocation VideoSender::GetAllocation(
- uint32_t bitrate_bps,
- uint32_t framerate_fps,
- VideoBitrateAllocator* bitrate_allocator) const {
- VideoBitrateAllocation bitrate_allocation;
- // Only call allocators if bitrate > 0 (ie, not suspended), otherwise they
- // might cap the bitrate to the min bitrate configured.
- if (bitrate_bps > 0) {
- if (bitrate_allocator) {
- bitrate_allocation =
- bitrate_allocator->GetAllocation(bitrate_bps, framerate_fps);
- } else {
- DefaultVideoBitrateAllocator default_allocator(current_codec_);
- bitrate_allocation =
- default_allocator.GetAllocation(bitrate_bps, framerate_fps);
- }
- }
- return bitrate_allocation;
-}
-
-void VideoSender::UpdateChannelParameters(
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback) {
- VideoBitrateAllocation target_rate;
- {
- rtc::CritScope cs(¶ms_crit_);
- encoder_params_ =
- UpdateEncoderParameters(encoder_params_, bitrate_allocator,
- encoder_params_.total_bitrate.bps());
- target_rate = encoder_params_.target_bitrate;
- }
- if (bitrate_updated_callback && target_rate.get_sum_bps() > 0) {
- bitrate_updated_callback->OnBitrateAllocationUpdated(target_rate);
- }
-}
-
int32_t VideoSender::SetChannelParameters(
- uint32_t target_bitrate_bps,
- VideoBitrateAllocator* bitrate_allocator,
- VideoBitrateAllocationObserver* bitrate_updated_callback) {
- EncoderParameters encoder_params;
- encoder_params = UpdateEncoderParameters(encoder_params, bitrate_allocator,
- target_bitrate_bps);
- if (bitrate_updated_callback && target_bitrate_bps > 0) {
- bitrate_updated_callback->OnBitrateAllocationUpdated(
- encoder_params.target_bitrate);
- }
-
+ const VideoBitrateAllocation& bitrate_allocation,
+ uint32_t framerate_fps) {
bool encoder_has_internal_source;
{
rtc::CritScope cs(¶ms_crit_);
- encoder_params_ = encoder_params;
encoder_has_internal_source = encoder_has_internal_source_;
}
- // For encoders with internal sources, we need to tell the encoder directly,
- // instead of waiting for an AddVideoFrame that will never come (internal
- // source encoders don't get input frames).
- if (encoder_has_internal_source) {
+ {
rtc::CritScope cs(&encoder_crit_);
if (_encoder) {
- SetEncoderParameters(encoder_params, encoder_has_internal_source);
+ // |target_bitrate == 0 | means that the network is down or the send pacer
+ // is full. We currently only report this if the encoder has an internal
+ // source. If the encoder does not have an internal source, higher levels
+ // are expected to not call AddVideoFrame. We do this since its unclear
+ // how current encoder implementations behave when given a zero target
+ // bitrate.
+ // TODO(perkj): Make sure all known encoder implementations handle zero
+ // target bitrate and remove this check.
+ if (!encoder_has_internal_source &&
+ bitrate_allocation.get_sum_bps() == 0) {
+ return VCM_OK;
+ }
+
+ if (framerate_fps == 0) {
+ // No frame rate estimate available, use default.
+ framerate_fps = current_codec_.maxFramerate;
+ }
+ if (_encoder != nullptr)
+ _encoder->SetEncoderParameters(bitrate_allocation, framerate_fps);
}
}
return VCM_OK;
}
-void VideoSender::SetEncoderParameters(EncoderParameters params,
- bool has_internal_source) {
- // |target_bitrate == 0 | means that the network is down or the send pacer is
- // full. We currently only report this if the encoder has an internal source.
- // If the encoder does not have an internal source, higher levels are expected
- // to not call AddVideoFrame. We do this since its unclear how current
- // encoder implementations behave when given a zero target bitrate.
- // TODO(perkj): Make sure all known encoder implementations handle zero
- // target bitrate and remove this check.
- if (!has_internal_source && params.target_bitrate.get_sum_bps() == 0)
- return;
-
- if (params.input_frame_rate == 0) {
- // No frame rate estimate available, use default.
- params.input_frame_rate = current_codec_.maxFramerate;
- }
- if (_encoder != nullptr)
- _encoder->SetEncoderParameters(params);
-}
-
// Add one raw video frame to the encoder, blocking.
int32_t VideoSender::AddVideoFrame(
const VideoFrame& videoFrame,
const CodecSpecificInfo* codecSpecificInfo,
absl::optional<VideoEncoder::EncoderInfo> encoder_info) {
- EncoderParameters encoder_params;
std::vector<FrameType> next_frame_types;
bool encoder_has_internal_source = false;
{
rtc::CritScope lock(¶ms_crit_);
- encoder_params = encoder_params_;
next_frame_types = next_frame_types_;
encoder_has_internal_source = encoder_has_internal_source_;
}
rtc::CritScope lock(&encoder_crit_);
if (_encoder == nullptr)
return VCM_UNINITIALIZED;
- SetEncoderParameters(encoder_params, encoder_has_internal_source);
- if (!encoder_info) {
- encoder_info = _encoder->GetEncoderInfo();
- }
-
- // Frame dropping is enabled iff frame dropping has been requested, and
- // frame dropping is not force-disabled, and rate controller is not trusted.
- const bool frame_dropping_enabled =
- !force_disable_frame_dropper_ &&
- !encoder_info->has_trusted_rate_controller;
- _mediaOpt.EnableFrameDropper(frame_dropping_enabled);
-
- if (_mediaOpt.DropFrame()) {
- RTC_LOG(LS_VERBOSE) << "Drop Frame: "
- << "target bitrate "
- << encoder_params.target_bitrate.get_sum_bps()
- << ", input frame rate "
- << encoder_params.input_frame_rate;
- post_encode_callback_->OnDroppedFrame(
- EncodedImageCallback::DropReason::kDroppedByMediaOptimizations);
- return VCM_OK;
- }
// TODO(pbos): Make sure setting send codec is synchronized with video
// processing so frame size always matches.
if (!_codecDataBase.MatchesCurrentResolution(videoFrame.width(),
diff --git a/modules/video_coding/video_sender_unittest.cc b/modules/video_coding/video_sender_unittest.cc
index 986bd3f..a9c9e45 100644
--- a/modules/video_coding/video_sender_unittest.cc
+++ b/modules/video_coding/video_sender_unittest.cc
@@ -319,14 +319,14 @@
SetRateAllocation(new_rate_allocation, settings_.maxFramerate))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_bitrate_kbps * 1000, rate_allocator_.get(),
- nullptr);
+ sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
AddFrame();
clock_.AdvanceTimeMilliseconds(kFrameIntervalMs);
// Expect no call to encoder_.SetRates if the new bitrate is zero.
EXPECT_CALL(encoder_, SetRateAllocation(_, _)).Times(0);
- sender_->SetChannelParameters(0, rate_allocator_.get(), nullptr);
+ sender_->SetChannelParameters(VideoBitrateAllocation(),
+ settings_.maxFramerate);
AddFrame();
}
@@ -363,8 +363,7 @@
EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, _))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_bitrate_kbps * 1000, rate_allocator_.get(),
- nullptr);
+ sender_->SetChannelParameters(new_rate_allocation, settings_.maxFramerate);
}
TEST_F(TestVideoSenderWithMockEncoder,
@@ -375,8 +374,9 @@
// Expect initial call to SetChannelParameters. Rates are initialized through
// InitEncode and expects no additional call before the framerate (or bitrate)
// updates.
- sender_->SetChannelParameters(settings_.startBitrate * 1000,
- rate_allocator_.get(), nullptr);
+ sender_->SetChannelParameters(
+ rate_allocator_->GetAllocation(settings_.startBitrate * 1000, kInputFps),
+ kInputFps);
while (clock_.TimeInMilliseconds() < start_time + kRateStatsWindowMs) {
AddFrame();
clock_.AdvanceTimeMilliseconds(1000 / kInputFps);
@@ -390,8 +390,7 @@
EXPECT_CALL(encoder_, SetRateAllocation(new_rate_allocation, kInputFps))
.Times(1)
.WillOnce(Return(0));
- sender_->SetChannelParameters(new_bitrate_bps, rate_allocator_.get(),
- nullptr);
+ sender_->SetChannelParameters(new_rate_allocation, kInputFps);
AddFrame();
}
@@ -439,12 +438,13 @@
AddFrame();
// SetChannelParameters needs to be called frequently to propagate
// framerate from the media optimization into the encoder.
- // Note: SetChannelParameters fails if less than 2 frames are in the
- // buffer since it will fail to calculate the framerate.
+ const VideoBitrateAllocation bitrate_allocation =
+ rate_allocator_->GetAllocation(available_bitrate_kbps_ * 1000,
+ static_cast<uint32_t>(framerate));
if (i != 0) {
- EXPECT_EQ(VCM_OK, sender_->SetChannelParameters(
- available_bitrate_kbps_ * 1000,
- rate_allocator_.get(), nullptr));
+ EXPECT_EQ(VCM_OK,
+ sender_->SetChannelParameters(
+ bitrate_allocation, static_cast<uint32_t>(framerate)));
}
}
}