blob: c1dac5ee44337e0a5f6dac1344b13fdd401b2c9f [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
stefan@webrtc.org07b45a52012-02-02 08:37:48 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "video/video_stream_encoder.h"
mflodman@webrtc.org84d17832011-12-01 17:02:23 +000012
stefan@webrtc.orgc3cc3752013-06-04 09:36:56 +000013#include <algorithm>
perkj57c21f92016-06-17 07:27:16 -070014#include <limits>
sprangc5d62e22017-04-02 23:53:04 -070015#include <numeric>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000017
Niels Möller6bb5ab92019-01-11 11:11:10 +010018#include "absl/memory/memory.h"
Niels Möller4dc66c52018-10-05 14:17:58 +020019#include "api/video/encoded_image.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "api/video/i420_buffer.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080021#include "api/video/video_bitrate_allocator_factory.h"
Sergey Silkin8b9b5f92018-12-10 09:28:53 +010022#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "modules/video_coding/include/video_codec_initializer.h"
24#include "modules/video_coding/include/video_coding.h"
Niels Möller6bb5ab92019-01-11 11:11:10 +010025#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "rtc_base/arraysize.h"
27#include "rtc_base/checks.h"
Erik Språng6a7baa72019-02-26 18:31:00 +010028#include "rtc_base/experiments/alr_experiment.h"
Åsa Perssona945aee2018-04-24 16:53:25 +020029#include "rtc_base/experiments/quality_scaling_experiment.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010030#include "rtc_base/experiments/rate_control_settings.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "rtc_base/location.h"
32#include "rtc_base/logging.h"
Jonas Olsson366a50c2018-09-06 13:41:30 +020033#include "rtc_base/strings/string_builder.h"
Karl Wiberg80ba3332018-02-05 10:33:35 +010034#include "rtc_base/system/fallthrough.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020036#include "rtc_base/trace_event.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020037#include "system_wrappers/include/field_trial.h"
nisseea3a7982017-05-15 02:42:11 -070038
niklase@google.com470e71d2011-07-07 08:21:25 +000039namespace webrtc {
40
perkj26091b12016-09-01 01:17:40 -070041namespace {
sprangb1ca0732017-02-01 08:38:12 -080042
asapersson6ffb67d2016-09-12 00:10:45 -070043// Time interval for logging frame counts.
44const int64_t kFrameLogIntervalMs = 60000;
sprangc5d62e22017-04-02 23:53:04 -070045const int kMinFramerateFps = 2;
perkj26091b12016-09-01 01:17:40 -070046
Sebastian Janssona3177052018-04-10 13:05:49 +020047// Time to keep a single cached pending frame in paused state.
48const int64_t kPendingFrameTimeoutMs = 1000;
49
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020050const char kInitialFramedropFieldTrial[] = "WebRTC-InitialFramedrop";
Niels Möller6bb5ab92019-01-11 11:11:10 +010051constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper";
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020052
kthelgason2bc68642017-02-07 07:02:22 -080053// The maximum number of frames to drop at beginning of stream
54// to try and achieve desired bitrate.
55const int kMaxInitialFramedrop = 4;
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020056// When the first change in BWE above this threshold occurs,
57// enable DropFrameDueToSize logic.
58const float kFramedropThreshold = 0.3;
kthelgason2bc68642017-02-07 07:02:22 -080059
Niels Möller6bb5ab92019-01-11 11:11:10 +010060// Averaging window spanning 90 frames at default 30fps, matching old media
61// optimization module defaults.
62const int64_t kFrameRateAvergingWindowSizeMs = (1000 / 30) * 90;
63
Erik Språngb7cb7b52019-02-26 15:52:33 +010064const size_t kDefaultPayloadSize = 1440;
65
Taylor Brandstetter49fcc102018-05-16 14:20:41 -070066// Initial limits for BALANCED degradation preference.
asaperssonf7e294d2017-06-13 23:25:22 -070067int MinFps(int pixels) {
68 if (pixels <= 320 * 240) {
69 return 7;
70 } else if (pixels <= 480 * 270) {
71 return 10;
72 } else if (pixels <= 640 * 480) {
73 return 15;
74 } else {
75 return std::numeric_limits<int>::max();
76 }
77}
78
79int MaxFps(int pixels) {
80 if (pixels <= 320 * 240) {
81 return 10;
82 } else if (pixels <= 480 * 270) {
83 return 15;
84 } else {
85 return std::numeric_limits<int>::max();
86 }
87}
88
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020089uint32_t abs_diff(uint32_t a, uint32_t b) {
90 return (a < b) ? b - a : a - b;
91}
92
Taylor Brandstetter49fcc102018-05-16 14:20:41 -070093bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) {
94 return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE ||
95 degradation_preference == DegradationPreference::BALANCED;
asapersson09f05612017-05-15 23:40:18 -070096}
97
Taylor Brandstetter49fcc102018-05-16 14:20:41 -070098bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) {
99 return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION ||
100 degradation_preference == DegradationPreference::BALANCED;
asapersson09f05612017-05-15 23:40:18 -0700101}
102
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200103// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
104// pipelining encoders better (multiple input frames before something comes
105// out). This should effectively turn off CPU adaptations for systems that
106// remotely cope with the load right now.
107CpuOveruseOptions GetCpuOveruseOptions(
Niels Möller213618e2018-07-24 09:29:58 +0200108 const VideoStreamEncoderSettings& settings,
Niels Möller4db138e2018-04-19 09:04:13 +0200109 bool full_overuse_time) {
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200110 CpuOveruseOptions options;
111
Niels Möller4db138e2018-04-19 09:04:13 +0200112 if (full_overuse_time) {
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200113 options.low_encode_usage_threshold_percent = 150;
114 options.high_encode_usage_threshold_percent = 200;
115 }
116 if (settings.experiment_cpu_load_estimator) {
117 options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec;
118 }
119
120 return options;
121}
122
Erik Språngb7cb7b52019-02-26 15:52:33 +0100123bool RequiresEncoderReset(const VideoCodec& previous_send_codec,
124 const VideoCodec& new_send_codec) {
125 // Does not check startBitrate or maxFramerate.
126 if (new_send_codec.codecType != previous_send_codec.codecType ||
127 new_send_codec.width != previous_send_codec.width ||
128 new_send_codec.height != previous_send_codec.height ||
129 new_send_codec.maxBitrate != previous_send_codec.maxBitrate ||
130 new_send_codec.minBitrate != previous_send_codec.minBitrate ||
131 new_send_codec.qpMax != previous_send_codec.qpMax ||
132 new_send_codec.numberOfSimulcastStreams !=
133 previous_send_codec.numberOfSimulcastStreams ||
134 new_send_codec.mode != previous_send_codec.mode) {
135 return true;
136 }
137
138 switch (new_send_codec.codecType) {
139 case kVideoCodecVP8:
140 if (new_send_codec.VP8() != previous_send_codec.VP8()) {
141 return true;
142 }
143 break;
144
145 case kVideoCodecVP9:
146 if (new_send_codec.VP9() != previous_send_codec.VP9()) {
147 return true;
148 }
149 break;
150
151 case kVideoCodecH264:
152 if (new_send_codec.H264() != previous_send_codec.H264()) {
153 return true;
154 }
155 break;
156
157 default:
158 break;
159 }
160
161 for (unsigned char i = 0; i < new_send_codec.numberOfSimulcastStreams; ++i) {
162 if (new_send_codec.simulcastStream[i] !=
163 previous_send_codec.simulcastStream[i])
164 return true;
165 }
166 return false;
167}
Erik Språng6a7baa72019-02-26 18:31:00 +0100168
169std::array<uint8_t, 2> GetExperimentGroups() {
170 std::array<uint8_t, 2> experiment_groups;
171 absl::optional<AlrExperimentSettings> experiment_settings =
172 AlrExperimentSettings::CreateFromFieldTrial(
173 AlrExperimentSettings::kStrictPacingAndProbingExperimentName);
174 if (experiment_settings) {
175 experiment_groups[0] = experiment_settings->group_id + 1;
176 } else {
177 experiment_groups[0] = 0;
178 }
179 experiment_settings = AlrExperimentSettings::CreateFromFieldTrial(
180 AlrExperimentSettings::kScreenshareProbingBweExperimentName);
181 if (experiment_settings) {
182 experiment_groups[1] = experiment_settings->group_id + 1;
183 } else {
184 experiment_groups[1] = 0;
185 }
186 return experiment_groups;
187}
perkj26091b12016-09-01 01:17:40 -0700188} // namespace
189
perkja49cbd32016-09-16 07:53:41 -0700190// VideoSourceProxy is responsible ensuring thread safety between calls to
mflodmancc3d4422017-08-03 08:27:51 -0700191// VideoStreamEncoder::SetSource that will happen on libjingle's worker thread
192// when a video capturer is connected to the encoder and the encoder task queue
perkja49cbd32016-09-16 07:53:41 -0700193// (encoder_queue_) where the encoder reports its VideoSinkWants.
mflodmancc3d4422017-08-03 08:27:51 -0700194class VideoStreamEncoder::VideoSourceProxy {
perkja49cbd32016-09-16 07:53:41 -0700195 public:
mflodmancc3d4422017-08-03 08:27:51 -0700196 explicit VideoSourceProxy(VideoStreamEncoder* video_stream_encoder)
197 : video_stream_encoder_(video_stream_encoder),
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700198 degradation_preference_(DegradationPreference::DISABLED),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200199 source_(nullptr),
200 max_framerate_(std::numeric_limits<int>::max()) {}
perkja49cbd32016-09-16 07:53:41 -0700201
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700202 void SetSource(rtc::VideoSourceInterface<VideoFrame>* source,
203 const DegradationPreference& degradation_preference) {
perkj803d97f2016-11-01 11:45:46 -0700204 // Called on libjingle's worker thread.
perkja49cbd32016-09-16 07:53:41 -0700205 RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_);
206 rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
perkj803d97f2016-11-01 11:45:46 -0700207 rtc::VideoSinkWants wants;
perkja49cbd32016-09-16 07:53:41 -0700208 {
209 rtc::CritScope lock(&crit_);
sprangc5d62e22017-04-02 23:53:04 -0700210 degradation_preference_ = degradation_preference;
perkja49cbd32016-09-16 07:53:41 -0700211 old_source = source_;
212 source_ = source;
sprangfda496a2017-06-15 04:21:07 -0700213 wants = GetActiveSinkWantsInternal();
perkja49cbd32016-09-16 07:53:41 -0700214 }
215
216 if (old_source != source && old_source != nullptr) {
mflodmancc3d4422017-08-03 08:27:51 -0700217 old_source->RemoveSink(video_stream_encoder_);
perkja49cbd32016-09-16 07:53:41 -0700218 }
219
220 if (!source) {
221 return;
222 }
223
mflodmancc3d4422017-08-03 08:27:51 -0700224 source->AddOrUpdateSink(video_stream_encoder_, wants);
perkja49cbd32016-09-16 07:53:41 -0700225 }
226
Åsa Persson8c1bf952018-09-13 10:42:19 +0200227 void SetMaxFramerate(int max_framerate) {
228 RTC_DCHECK_GT(max_framerate, 0);
229 rtc::CritScope lock(&crit_);
230 if (max_framerate == max_framerate_)
231 return;
232
233 RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate;
234 max_framerate_ = max_framerate;
235 if (source_) {
236 source_->AddOrUpdateSink(video_stream_encoder_,
237 GetActiveSinkWantsInternal());
238 }
239 }
240
perkj803d97f2016-11-01 11:45:46 -0700241 void SetWantsRotationApplied(bool rotation_applied) {
242 rtc::CritScope lock(&crit_);
243 sink_wants_.rotation_applied = rotation_applied;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200244 if (source_) {
245 source_->AddOrUpdateSink(video_stream_encoder_,
246 GetActiveSinkWantsInternal());
247 }
sprangc5d62e22017-04-02 23:53:04 -0700248 }
249
sprangfda496a2017-06-15 04:21:07 -0700250 rtc::VideoSinkWants GetActiveSinkWants() {
251 rtc::CritScope lock(&crit_);
252 return GetActiveSinkWantsInternal();
perkj803d97f2016-11-01 11:45:46 -0700253 }
254
asaperssonf7e294d2017-06-13 23:25:22 -0700255 void ResetPixelFpsCount() {
256 rtc::CritScope lock(&crit_);
257 sink_wants_.max_pixel_count = std::numeric_limits<int>::max();
258 sink_wants_.target_pixel_count.reset();
259 sink_wants_.max_framerate_fps = std::numeric_limits<int>::max();
260 if (source_)
Åsa Persson8c1bf952018-09-13 10:42:19 +0200261 source_->AddOrUpdateSink(video_stream_encoder_,
262 GetActiveSinkWantsInternal());
asaperssonf7e294d2017-06-13 23:25:22 -0700263 }
264
Åsa Perssonc3ed6302017-11-16 14:04:52 +0100265 bool RequestResolutionLowerThan(int pixel_count,
266 int min_pixels_per_frame,
267 bool* min_pixels_reached) {
perkj803d97f2016-11-01 11:45:46 -0700268 // Called on the encoder task queue.
269 rtc::CritScope lock(&crit_);
asapersson13874762017-06-07 00:01:02 -0700270 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
asapersson02465b82017-04-10 01:12:52 -0700271 // This can happen since |degradation_preference_| is set on libjingle's
272 // worker thread but the adaptation is done on the encoder task queue.
asaperssond0de2952017-04-21 01:47:31 -0700273 return false;
perkj803d97f2016-11-01 11:45:46 -0700274 }
asapersson13874762017-06-07 00:01:02 -0700275 // The input video frame size will have a resolution less than or equal to
276 // |max_pixel_count| depending on how the source can scale the frame size.
kthelgason5e13d412016-12-01 03:59:51 -0800277 const int pixels_wanted = (pixel_count * 3) / 5;
Åsa Perssonc3ed6302017-11-16 14:04:52 +0100278 if (pixels_wanted >= sink_wants_.max_pixel_count) {
279 return false;
280 }
281 if (pixels_wanted < min_pixels_per_frame) {
282 *min_pixels_reached = true;
asaperssond0de2952017-04-21 01:47:31 -0700283 return false;
asapersson13874762017-06-07 00:01:02 -0700284 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100285 RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: "
286 << pixels_wanted;
sprangc5d62e22017-04-02 23:53:04 -0700287 sink_wants_.max_pixel_count = pixels_wanted;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200288 sink_wants_.target_pixel_count = absl::nullopt;
mflodmancc3d4422017-08-03 08:27:51 -0700289 source_->AddOrUpdateSink(video_stream_encoder_,
290 GetActiveSinkWantsInternal());
asaperssond0de2952017-04-21 01:47:31 -0700291 return true;
sprangc5d62e22017-04-02 23:53:04 -0700292 }
293
sprangfda496a2017-06-15 04:21:07 -0700294 int RequestFramerateLowerThan(int fps) {
sprangc5d62e22017-04-02 23:53:04 -0700295 // Called on the encoder task queue.
asapersson13874762017-06-07 00:01:02 -0700296 // The input video frame rate will be scaled down to 2/3, rounding down.
sprangfda496a2017-06-15 04:21:07 -0700297 int framerate_wanted = (fps * 2) / 3;
298 return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
perkj803d97f2016-11-01 11:45:46 -0700299 }
300
asapersson13874762017-06-07 00:01:02 -0700301 bool RequestHigherResolutionThan(int pixel_count) {
302 // Called on the encoder task queue.
perkj803d97f2016-11-01 11:45:46 -0700303 rtc::CritScope lock(&crit_);
asapersson13874762017-06-07 00:01:02 -0700304 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
asapersson02465b82017-04-10 01:12:52 -0700305 // This can happen since |degradation_preference_| is set on libjingle's
306 // worker thread but the adaptation is done on the encoder task queue.
asapersson13874762017-06-07 00:01:02 -0700307 return false;
perkj803d97f2016-11-01 11:45:46 -0700308 }
asapersson13874762017-06-07 00:01:02 -0700309 int max_pixels_wanted = pixel_count;
310 if (max_pixels_wanted != std::numeric_limits<int>::max())
311 max_pixels_wanted = pixel_count * 4;
sprangc5d62e22017-04-02 23:53:04 -0700312
asapersson13874762017-06-07 00:01:02 -0700313 if (max_pixels_wanted <= sink_wants_.max_pixel_count)
314 return false;
315
316 sink_wants_.max_pixel_count = max_pixels_wanted;
317 if (max_pixels_wanted == std::numeric_limits<int>::max()) {
sprangc5d62e22017-04-02 23:53:04 -0700318 // Remove any constraints.
319 sink_wants_.target_pixel_count.reset();
sprangc5d62e22017-04-02 23:53:04 -0700320 } else {
321 // On step down we request at most 3/5 the pixel count of the previous
322 // resolution, so in order to take "one step up" we request a resolution
323 // as close as possible to 5/3 of the current resolution. The actual pixel
324 // count selected depends on the capabilities of the source. In order to
325 // not take a too large step up, we cap the requested pixel count to be at
326 // most four time the current number of pixels.
Oskar Sundbom8e07c132018-01-08 16:45:42 +0100327 sink_wants_.target_pixel_count = (pixel_count * 5) / 3;
sprangc5d62e22017-04-02 23:53:04 -0700328 }
Mirko Bonadei675513b2017-11-09 11:09:25 +0100329 RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: "
330 << max_pixels_wanted;
mflodmancc3d4422017-08-03 08:27:51 -0700331 source_->AddOrUpdateSink(video_stream_encoder_,
332 GetActiveSinkWantsInternal());
asapersson13874762017-06-07 00:01:02 -0700333 return true;
sprangc5d62e22017-04-02 23:53:04 -0700334 }
335
sprangfda496a2017-06-15 04:21:07 -0700336 // Request upgrade in framerate. Returns the new requested frame, or -1 if
337 // no change requested. Note that maxint may be returned if limits due to
338 // adaptation requests are removed completely. In that case, consider
339 // |max_framerate_| to be the current limit (assuming the capturer complies).
340 int RequestHigherFramerateThan(int fps) {
asapersson13874762017-06-07 00:01:02 -0700341 // Called on the encoder task queue.
342 // The input frame rate will be scaled up to the last step, with rounding.
343 int framerate_wanted = fps;
344 if (fps != std::numeric_limits<int>::max())
345 framerate_wanted = (fps * 3) / 2;
346
sprangfda496a2017-06-15 04:21:07 -0700347 return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1;
asapersson13874762017-06-07 00:01:02 -0700348 }
349
350 bool RestrictFramerate(int fps) {
sprangc5d62e22017-04-02 23:53:04 -0700351 // Called on the encoder task queue.
352 rtc::CritScope lock(&crit_);
asapersson13874762017-06-07 00:01:02 -0700353 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
354 return false;
355
356 const int fps_wanted = std::max(kMinFramerateFps, fps);
357 if (fps_wanted >= sink_wants_.max_framerate_fps)
358 return false;
359
Mirko Bonadei675513b2017-11-09 11:09:25 +0100360 RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted;
asapersson13874762017-06-07 00:01:02 -0700361 sink_wants_.max_framerate_fps = fps_wanted;
mflodmancc3d4422017-08-03 08:27:51 -0700362 source_->AddOrUpdateSink(video_stream_encoder_,
363 GetActiveSinkWantsInternal());
asapersson13874762017-06-07 00:01:02 -0700364 return true;
365 }
366
367 bool IncreaseFramerate(int fps) {
368 // Called on the encoder task queue.
369 rtc::CritScope lock(&crit_);
370 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
371 return false;
372
373 const int fps_wanted = std::max(kMinFramerateFps, fps);
374 if (fps_wanted <= sink_wants_.max_framerate_fps)
375 return false;
376
Mirko Bonadei675513b2017-11-09 11:09:25 +0100377 RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted;
asapersson13874762017-06-07 00:01:02 -0700378 sink_wants_.max_framerate_fps = fps_wanted;
mflodmancc3d4422017-08-03 08:27:51 -0700379 source_->AddOrUpdateSink(video_stream_encoder_,
380 GetActiveSinkWantsInternal());
asapersson13874762017-06-07 00:01:02 -0700381 return true;
perkj803d97f2016-11-01 11:45:46 -0700382 }
383
perkja49cbd32016-09-16 07:53:41 -0700384 private:
sprangfda496a2017-06-15 04:21:07 -0700385 rtc::VideoSinkWants GetActiveSinkWantsInternal()
danilchapa37de392017-09-09 04:17:22 -0700386 RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
sprangfda496a2017-06-15 04:21:07 -0700387 rtc::VideoSinkWants wants = sink_wants_;
388 // Clear any constraints from the current sink wants that don't apply to
389 // the used degradation_preference.
390 switch (degradation_preference_) {
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700391 case DegradationPreference::BALANCED:
sprangfda496a2017-06-15 04:21:07 -0700392 break;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700393 case DegradationPreference::MAINTAIN_FRAMERATE:
sprangfda496a2017-06-15 04:21:07 -0700394 wants.max_framerate_fps = std::numeric_limits<int>::max();
395 break;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700396 case DegradationPreference::MAINTAIN_RESOLUTION:
sprangfda496a2017-06-15 04:21:07 -0700397 wants.max_pixel_count = std::numeric_limits<int>::max();
398 wants.target_pixel_count.reset();
399 break;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700400 case DegradationPreference::DISABLED:
sprangfda496a2017-06-15 04:21:07 -0700401 wants.max_pixel_count = std::numeric_limits<int>::max();
402 wants.target_pixel_count.reset();
403 wants.max_framerate_fps = std::numeric_limits<int>::max();
404 }
Åsa Persson8c1bf952018-09-13 10:42:19 +0200405 // Limit to configured max framerate.
406 wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps);
sprangfda496a2017-06-15 04:21:07 -0700407 return wants;
408 }
409
perkja49cbd32016-09-16 07:53:41 -0700410 rtc::CriticalSection crit_;
411 rtc::SequencedTaskChecker main_checker_;
mflodmancc3d4422017-08-03 08:27:51 -0700412 VideoStreamEncoder* const video_stream_encoder_;
danilchapa37de392017-09-09 04:17:22 -0700413 rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700414 DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_);
danilchapa37de392017-09-09 04:17:22 -0700415 rtc::VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&crit_);
Åsa Persson8c1bf952018-09-13 10:42:19 +0200416 int max_framerate_ RTC_GUARDED_BY(&crit_);
perkja49cbd32016-09-16 07:53:41 -0700417
418 RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
419};
420
Åsa Persson0122e842017-10-16 12:19:23 +0200421VideoStreamEncoder::VideoStreamEncoder(
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100422 Clock* clock,
Åsa Persson0122e842017-10-16 12:19:23 +0200423 uint32_t number_of_cores,
Niels Möller213618e2018-07-24 09:29:58 +0200424 VideoStreamEncoderObserver* encoder_stats_observer,
425 const VideoStreamEncoderSettings& settings,
Sebastian Jansson74682c12019-03-01 11:50:20 +0100426 std::unique_ptr<OveruseFrameDetector> overuse_detector,
427 TaskQueueFactory* task_queue_factory)
perkj26091b12016-09-01 01:17:40 -0700428 : shutdown_event_(true /* manual_reset */, false),
429 number_of_cores_(number_of_cores),
Kári Tristan Helgason639602a2018-08-02 10:51:40 +0200430 initial_framedrop_(0),
431 initial_framedrop_on_bwe_enabled_(
432 webrtc::field_trial::IsEnabled(kInitialFramedropFieldTrial)),
Åsa Perssona945aee2018-04-24 16:53:25 +0200433 quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()),
perkja49cbd32016-09-16 07:53:41 -0700434 source_proxy_(new VideoSourceProxy(this)),
Pera48ddb72016-09-29 11:48:50 +0200435 sink_(nullptr),
perkj26091b12016-09-01 01:17:40 -0700436 settings_(settings),
Erik Språng7ca375c2019-02-06 16:20:17 +0100437 rate_control_settings_(RateControlSettings::ParseFromFieldTrials()),
Niels Möller73f29cb2018-01-31 16:09:31 +0100438 overuse_detector_(std::move(overuse_detector)),
Niels Möller213618e2018-07-24 09:29:58 +0200439 encoder_stats_observer_(encoder_stats_observer),
Erik Språng6a7baa72019-02-26 18:31:00 +0100440 encoder_initialized_(false),
sprangfda496a2017-06-15 04:21:07 -0700441 max_framerate_(-1),
perkjfa10b552016-10-02 23:45:26 -0700442 pending_encoder_reconfiguration_(false),
Mirta Dvornicic1ec2a162018-12-10 09:47:34 +0000443 pending_encoder_creation_(false),
Erik Språnge2fd86a2018-10-24 11:32:39 +0200444 crop_width_(0),
445 crop_height_(0),
perkj26091b12016-09-01 01:17:40 -0700446 encoder_start_bitrate_bps_(0),
Pera48ddb72016-09-29 11:48:50 +0200447 max_data_payload_length_(0),
pbos@webrtc.org143451d2015-03-18 14:40:03 +0000448 last_observed_bitrate_bps_(0),
stefan@webrtc.org792f1a12015-03-04 12:24:26 +0000449 encoder_paused_and_dropped_frame_(false),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100450 clock_(clock),
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700451 degradation_preference_(DegradationPreference::DISABLED),
Yuwei Huangd9f99c12017-10-24 15:40:52 -0700452 posted_frames_waiting_for_encode_(0),
perkj26091b12016-09-01 01:17:40 -0700453 last_captured_timestamp_(0),
454 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
455 clock_->TimeInMilliseconds()),
asapersson6ffb67d2016-09-12 00:10:45 -0700456 last_frame_log_ms_(clock_->TimeInMilliseconds()),
457 captured_frame_count_(0),
458 dropped_frame_count_(0),
Erik Språnge2fd86a2018-10-24 11:32:39 +0200459 pending_frame_post_time_us_(0),
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100460 accumulated_update_rect_{0, 0, 0, 0},
sprang1a646ee2016-12-01 06:34:11 -0800461 bitrate_observer_(nullptr),
Erik Språng6a7baa72019-02-26 18:31:00 +0100462 last_framerate_fps_(0),
Niels Möller6bb5ab92019-01-11 11:11:10 +0100463 force_disable_frame_dropper_(false),
464 input_framerate_(kFrameRateAvergingWindowSizeMs, 1000),
465 pending_frame_drops_(0),
Erik Språngd7329ca2019-02-21 21:19:53 +0100466 next_frame_types_(1, kVideoFrameDelta),
Erik Språng6a7baa72019-02-26 18:31:00 +0100467 frame_encoder_timer_(this),
468 experiment_groups_(GetExperimentGroups()),
Sebastian Jansson74682c12019-03-01 11:50:20 +0100469 encoder_queue_(task_queue_factory->CreateTaskQueue(
470 "EncoderQueue",
471 TaskQueueFactory::Priority::NORMAL)) {
Niels Möller213618e2018-07-24 09:29:58 +0200472 RTC_DCHECK(encoder_stats_observer);
Niels Möller73f29cb2018-01-31 16:09:31 +0100473 RTC_DCHECK(overuse_detector_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100474 RTC_DCHECK_GE(number_of_cores, 1);
mflodman@webrtc.org02270cd2015-02-06 13:10:19 +0000475}
476
mflodmancc3d4422017-08-03 08:27:51 -0700477VideoStreamEncoder::~VideoStreamEncoder() {
perkja49cbd32016-09-16 07:53:41 -0700478 RTC_DCHECK_RUN_ON(&thread_checker_);
perkj26091b12016-09-01 01:17:40 -0700479 RTC_DCHECK(shutdown_event_.Wait(0))
480 << "Must call ::Stop() before destruction.";
481}
482
mflodmancc3d4422017-08-03 08:27:51 -0700483void VideoStreamEncoder::Stop() {
perkja49cbd32016-09-16 07:53:41 -0700484 RTC_DCHECK_RUN_ON(&thread_checker_);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700485 source_proxy_->SetSource(nullptr, DegradationPreference());
perkja49cbd32016-09-16 07:53:41 -0700486 encoder_queue_.PostTask([this] {
487 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangfda496a2017-06-15 04:21:07 -0700488 overuse_detector_->StopCheckForOveruse();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100489 rate_allocator_ = nullptr;
sprang1a646ee2016-12-01 06:34:11 -0800490 bitrate_observer_ = nullptr;
Erik Språng6a7baa72019-02-26 18:31:00 +0100491 ReleaseEncoder();
kthelgason876222f2016-11-29 01:44:11 -0800492 quality_scaler_ = nullptr;
perkja49cbd32016-09-16 07:53:41 -0700493 shutdown_event_.Set();
494 });
495
496 shutdown_event_.Wait(rtc::Event::kForever);
perkj26091b12016-09-01 01:17:40 -0700497}
498
Niels Möller0327c2d2018-05-21 14:09:31 +0200499void VideoStreamEncoder::SetBitrateAllocationObserver(
sprang1a646ee2016-12-01 06:34:11 -0800500 VideoBitrateAllocationObserver* bitrate_observer) {
501 RTC_DCHECK_RUN_ON(&thread_checker_);
502 encoder_queue_.PostTask([this, bitrate_observer] {
503 RTC_DCHECK_RUN_ON(&encoder_queue_);
504 RTC_DCHECK(!bitrate_observer_);
505 bitrate_observer_ = bitrate_observer;
506 });
507}
508
mflodmancc3d4422017-08-03 08:27:51 -0700509void VideoStreamEncoder::SetSource(
perkj803d97f2016-11-01 11:45:46 -0700510 rtc::VideoSourceInterface<VideoFrame>* source,
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700511 const DegradationPreference& degradation_preference) {
perkja49cbd32016-09-16 07:53:41 -0700512 RTC_DCHECK_RUN_ON(&thread_checker_);
perkj803d97f2016-11-01 11:45:46 -0700513 source_proxy_->SetSource(source, degradation_preference);
514 encoder_queue_.PostTask([this, degradation_preference] {
515 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangc5d62e22017-04-02 23:53:04 -0700516 if (degradation_preference_ != degradation_preference) {
517 // Reset adaptation state, so that we're not tricked into thinking there's
518 // an already pending request of the same type.
519 last_adaptation_request_.reset();
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700520 if (degradation_preference == DegradationPreference::BALANCED ||
521 degradation_preference_ == DegradationPreference::BALANCED) {
asaperssonf7e294d2017-06-13 23:25:22 -0700522 // TODO(asapersson): Consider removing |adapt_counters_| map and use one
523 // AdaptCounter for all modes.
524 source_proxy_->ResetPixelFpsCount();
525 adapt_counters_.clear();
526 }
sprangc5d62e22017-04-02 23:53:04 -0700527 }
sprangb1ca0732017-02-01 08:38:12 -0800528 degradation_preference_ = degradation_preference;
Niels Möller4db138e2018-04-19 09:04:13 +0200529
Mirta Dvornicic1ec2a162018-12-10 09:47:34 +0000530 if (encoder_)
Erik Språng7ca375c2019-02-06 16:20:17 +0100531 ConfigureQualityScaler(encoder_->GetEncoderInfo());
Niels Möller4db138e2018-04-19 09:04:13 +0200532
Niels Möller7dc26b72017-12-06 10:27:48 +0100533 if (!IsFramerateScalingEnabled(degradation_preference) &&
534 max_framerate_ != -1) {
535 // If frame rate scaling is no longer allowed, remove any potential
536 // allowance for longer frame intervals.
537 overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
538 }
perkj803d97f2016-11-01 11:45:46 -0700539 });
perkja49cbd32016-09-16 07:53:41 -0700540}
541
mflodmancc3d4422017-08-03 08:27:51 -0700542void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
perkj803d97f2016-11-01 11:45:46 -0700543 source_proxy_->SetWantsRotationApplied(rotation_applied);
perkj26091b12016-09-01 01:17:40 -0700544 encoder_queue_.PostTask([this, sink] {
545 RTC_DCHECK_RUN_ON(&encoder_queue_);
546 sink_ = sink;
547 });
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000548}
549
mflodmancc3d4422017-08-03 08:27:51 -0700550void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) {
perkj26091b12016-09-01 01:17:40 -0700551 encoder_queue_.PostTask([this, start_bitrate_bps] {
552 RTC_DCHECK_RUN_ON(&encoder_queue_);
553 encoder_start_bitrate_bps_ = start_bitrate_bps;
554 });
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000555}
Peter Boström00b9d212016-05-19 16:59:03 +0200556
mflodmancc3d4422017-08-03 08:27:51 -0700557void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config,
Niels Möllerf1338562018-04-26 09:51:47 +0200558 size_t max_data_payload_length) {
Sebastian Jansson3dc01252018-03-19 19:27:44 +0100559 // TODO(srte): This struct should be replaced by a lambda with move capture
560 // when C++14 lambda is allowed.
561 struct ConfigureEncoderTask {
562 void operator()() {
Yves Gerey665174f2018-06-19 15:03:05 +0200563 encoder->ConfigureEncoderOnTaskQueue(std::move(config),
564 max_data_payload_length);
Sebastian Jansson3dc01252018-03-19 19:27:44 +0100565 }
566 VideoStreamEncoder* encoder;
567 VideoEncoderConfig config;
568 size_t max_data_payload_length;
Sebastian Jansson3dc01252018-03-19 19:27:44 +0100569 };
Yves Gerey665174f2018-06-19 15:03:05 +0200570 encoder_queue_.PostTask(
571 ConfigureEncoderTask{this, std::move(config), max_data_payload_length});
perkj26091b12016-09-01 01:17:40 -0700572}
573
mflodmancc3d4422017-08-03 08:27:51 -0700574void VideoStreamEncoder::ConfigureEncoderOnTaskQueue(
575 VideoEncoderConfig config,
Niels Möllerf1338562018-04-26 09:51:47 +0200576 size_t max_data_payload_length) {
perkj26091b12016-09-01 01:17:40 -0700577 RTC_DCHECK_RUN_ON(&encoder_queue_);
perkj26091b12016-09-01 01:17:40 -0700578 RTC_DCHECK(sink_);
Mirko Bonadei675513b2017-11-09 11:09:25 +0100579 RTC_LOG(LS_INFO) << "ConfigureEncoder requested.";
Pera48ddb72016-09-29 11:48:50 +0200580
Mirta Dvornicic1ec2a162018-12-10 09:47:34 +0000581 pending_encoder_creation_ =
Erik Språngd7329ca2019-02-21 21:19:53 +0100582 (!encoder_ || encoder_config_.video_format != config.video_format ||
583 max_data_payload_length_ != max_data_payload_length);
Pera48ddb72016-09-29 11:48:50 +0200584 encoder_config_ = std::move(config);
Erik Språngd7329ca2019-02-21 21:19:53 +0100585 max_data_payload_length_ = max_data_payload_length;
perkjfa10b552016-10-02 23:45:26 -0700586 pending_encoder_reconfiguration_ = true;
Pera48ddb72016-09-29 11:48:50 +0200587
perkjfa10b552016-10-02 23:45:26 -0700588 // Reconfigure the encoder now if the encoder has an internal source or
Per21d45d22016-10-30 21:37:57 +0100589 // if the frame resolution is known. Otherwise, the reconfiguration is
590 // deferred until the next frame to minimize the number of reconfigurations.
591 // The codec configuration depends on incoming video frame size.
592 if (last_frame_info_) {
593 ReconfigureEncoder();
Erik Språngd7329ca2019-02-21 21:19:53 +0100594 } else {
595 codec_info_ = settings_.encoder_factory->QueryVideoEncoder(
596 encoder_config_.video_format);
597 if (HasInternalSource()) {
598 last_frame_info_ = VideoFrameInfo(176, 144, false);
599 ReconfigureEncoder();
600 }
perkjfa10b552016-10-02 23:45:26 -0700601 }
602}
perkj26091b12016-09-01 01:17:40 -0700603
Seth Hampsoncc7125f2018-02-02 08:46:16 -0800604// TODO(bugs.webrtc.org/8807): Currently this always does a hard
605// reconfiguration, but this isn't always necessary. Add in logic to only update
606// the VideoBitrateAllocator and call OnEncoderConfigurationChanged with a
607// "soft" reconfiguration.
mflodmancc3d4422017-08-03 08:27:51 -0700608void VideoStreamEncoder::ReconfigureEncoder() {
perkjfa10b552016-10-02 23:45:26 -0700609 RTC_DCHECK(pending_encoder_reconfiguration_);
610 std::vector<VideoStream> streams =
611 encoder_config_.video_stream_factory->CreateEncoderStreams(
612 last_frame_info_->width, last_frame_info_->height, encoder_config_);
perkj26091b12016-09-01 01:17:40 -0700613
ilnik6b826ef2017-06-16 06:53:48 -0700614 // TODO(ilnik): If configured resolution is significantly less than provided,
615 // e.g. because there are not enough SSRCs for all simulcast streams,
616 // signal new resolutions via SinkWants to video source.
617
618 // Stream dimensions may be not equal to given because of a simulcast
619 // restrictions.
Florent Castelli450b5482018-11-29 17:32:47 +0100620 auto highest_stream = std::max_element(
621 streams.begin(), streams.end(),
622 [](const webrtc::VideoStream& a, const webrtc::VideoStream& b) {
623 return std::tie(a.width, a.height) < std::tie(b.width, b.height);
624 });
625 int highest_stream_width = static_cast<int>(highest_stream->width);
626 int highest_stream_height = static_cast<int>(highest_stream->height);
ilnik6b826ef2017-06-16 06:53:48 -0700627 // Dimension may be reduced to be, e.g. divisible by 4.
628 RTC_CHECK_GE(last_frame_info_->width, highest_stream_width);
629 RTC_CHECK_GE(last_frame_info_->height, highest_stream_height);
630 crop_width_ = last_frame_info_->width - highest_stream_width;
631 crop_height_ = last_frame_info_->height - highest_stream_height;
632
Erik Språng08127a92016-11-16 16:41:30 +0100633 VideoCodec codec;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800634 if (!VideoCodecInitializer::SetupCodec(encoder_config_, streams, &codec)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100635 RTC_LOG(LS_ERROR) << "Failed to create encoder configuration.";
Erik Språng08127a92016-11-16 16:41:30 +0100636 }
perkjfa10b552016-10-02 23:45:26 -0700637
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800638 rate_allocator_ =
639 settings_.bitrate_allocator_factory->CreateVideoBitrateAllocator(codec);
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800640
“Michael277a6562018-06-01 14:09:19 -0500641 // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9.
642 if (encoder_config_.codec_type == kVideoCodecVP9) {
“Michael277a6562018-06-01 14:09:19 -0500643 // Lower max bitrate to the level codec actually can produce.
Sergey Silkin8b9b5f92018-12-10 09:28:53 +0100644 streams[0].max_bitrate_bps = std::min<int>(
645 streams[0].max_bitrate_bps, SvcRateAllocator::GetMaxBitrateBps(codec));
“Michael277a6562018-06-01 14:09:19 -0500646 streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000;
Sergey Silkin8b9b5f92018-12-10 09:28:53 +0100647 // target_bitrate_bps specifies the maximum padding bitrate.
“Michael277a6562018-06-01 14:09:19 -0500648 streams[0].target_bitrate_bps =
Sergey Silkin8b9b5f92018-12-10 09:28:53 +0100649 SvcRateAllocator::GetPaddingBitrateBps(codec);
“Michael277a6562018-06-01 14:09:19 -0500650 }
651
perkjfa10b552016-10-02 23:45:26 -0700652 codec.startBitrate =
653 std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
654 codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
655 codec.expect_encode_from_texture = last_frame_info_->is_texture;
Erik Språngd7329ca2019-02-21 21:19:53 +0100656 // Make sure the start bit rate is sane...
657 RTC_DCHECK_LE(codec.startBitrate, 1000000);
sprangfda496a2017-06-15 04:21:07 -0700658 max_framerate_ = codec.maxFramerate;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200659
660 // Inform source about max configured framerate.
661 int max_framerate = 0;
662 for (const auto& stream : streams) {
663 max_framerate = std::max(stream.max_framerate, max_framerate);
664 }
665 source_proxy_->SetMaxFramerate(max_framerate);
Stefan Holmere5904162015-03-26 11:11:06 +0100666
Erik Språngb7cb7b52019-02-26 15:52:33 +0100667 if (codec.maxBitrate == 0) {
668 // max is one bit per pixel
669 codec.maxBitrate =
670 (static_cast<int>(codec.height) * static_cast<int>(codec.width) *
671 static_cast<int>(codec.maxFramerate)) /
672 1000;
673 if (codec.startBitrate > codec.maxBitrate) {
674 // But if the user tries to set a higher start bit rate we will
675 // increase the max accordingly.
676 codec.maxBitrate = codec.startBitrate;
677 }
678 }
679
680 if (codec.startBitrate > codec.maxBitrate) {
681 codec.startBitrate = codec.maxBitrate;
682 }
683
684 // Reset (release existing encoder) if one exists and anything except
685 // start bitrate or max framerate has changed. Don't call Release() if
686 // |pending_encoder_creation_| as that means this is a new encoder
687 // that has not yet been initialized.
688 const bool reset_required = RequiresEncoderReset(codec, send_codec_);
689 send_codec_ = codec;
690
Niels Möller4db138e2018-04-19 09:04:13 +0200691 // Keep the same encoder, as long as the video_format is unchanged.
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100692 // Encoder creation block is split in two since EncoderInfo needed to start
693 // CPU adaptation with the correct settings should be polled after
694 // encoder_->InitEncode().
Erik Språngb7cb7b52019-02-26 15:52:33 +0100695 bool success = true;
Erik Språng6a7baa72019-02-26 18:31:00 +0100696 if (pending_encoder_creation_ || reset_required) {
697 ReleaseEncoder();
698 if (pending_encoder_creation_) {
699 encoder_ = settings_.encoder_factory->CreateVideoEncoder(
700 encoder_config_.video_format);
701 // TODO(nisse): What to do if creating the encoder fails? Crash,
702 // or just discard incoming frames?
703 RTC_CHECK(encoder_);
704 codec_info_ = settings_.encoder_factory->QueryVideoEncoder(
705 encoder_config_.video_format);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100706 }
Erik Språng6a7baa72019-02-26 18:31:00 +0100707
708 if (encoder_->InitEncode(&send_codec_, number_of_cores_,
709 max_data_payload_length_ > 0
710 ? max_data_payload_length_
711 : kDefaultPayloadSize) != 0) {
712 RTC_LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
713 "codec type: "
714 << CodecTypeToPayloadString(send_codec_.codecType)
715 << " (" << send_codec_.codecType << ")";
716 ReleaseEncoder();
717 success = false;
718 } else {
719 encoder_initialized_ = true;
720 encoder_->RegisterEncodeCompleteCallback(this);
721 frame_encoder_timer_.OnEncoderInit(send_codec_, HasInternalSource());
722 }
723
724 frame_encoder_timer_.Reset();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100725 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100726
727 if (success) {
Erik Språngd7329ca2019-02-21 21:19:53 +0100728 next_frame_types_.clear();
729 next_frame_types_.resize(
730 std::max(static_cast<int>(codec.numberOfSimulcastStreams), 1),
731 kVideoFrameKey);
732 RTC_LOG(LS_VERBOSE) << " max bitrate " << codec.maxBitrate
733 << " start bitrate " << codec.startBitrate
734 << " max frame rate " << codec.maxFramerate
735 << " max payload size " << max_data_payload_length_;
736 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100737 RTC_LOG(LS_ERROR) << "Failed to configure encoder.";
Erik Språngb7cb7b52019-02-26 15:52:33 +0100738 rate_allocator_ = nullptr;
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000739 }
Peter Boström905f8e72016-03-02 16:59:56 +0100740
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100741 if (pending_encoder_creation_) {
742 overuse_detector_->StopCheckForOveruse();
743 overuse_detector_->StartCheckForOveruse(
Sebastian Janssoncda86dd2019-03-11 17:26:36 +0100744 &encoder_queue_,
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100745 GetCpuOveruseOptions(
746 settings_, encoder_->GetEncoderInfo().is_hardware_accelerated),
747 this);
748 pending_encoder_creation_ = false;
749 }
750
Niels Möller6bb5ab92019-01-11 11:11:10 +0100751 int num_layers;
752 if (codec.codecType == kVideoCodecVP8) {
753 num_layers = codec.VP8()->numberOfTemporalLayers;
754 } else if (codec.codecType == kVideoCodecVP9) {
755 num_layers = codec.VP9()->numberOfTemporalLayers;
Johnny Lee1a1c52b2019-02-08 14:25:40 -0500756 } else if (codec.codecType == kVideoCodecH264) {
757 num_layers = codec.H264()->numberOfTemporalLayers;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100758 } else if (codec.codecType == kVideoCodecGeneric &&
759 codec.numberOfSimulcastStreams > 0) {
760 // This is mainly for unit testing, disabling frame dropping.
761 // TODO(sprang): Add a better way to disable frame dropping.
762 num_layers = codec.simulcastStream[0].numberOfTemporalLayers;
763 } else {
764 num_layers = 1;
765 }
766
767 frame_dropper_.Reset();
768 frame_dropper_.SetRates(codec.startBitrate, max_framerate_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100769 // Force-disable frame dropper if either:
770 // * We have screensharing with layers.
771 // * "WebRTC-FrameDropper" field trial is "Disabled".
772 force_disable_frame_dropper_ =
773 field_trial::IsDisabled(kFrameDropperFieldTrial) ||
774 (num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing);
775
Erik Språng7ca375c2019-02-06 16:20:17 +0100776 VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
777 if (rate_control_settings_.UseEncoderBitrateAdjuster()) {
778 bitrate_adjuster_ = absl::make_unique<EncoderBitrateAdjuster>(codec);
779 bitrate_adjuster_->OnEncoderInfo(info);
780 }
781
Niels Möller6bb5ab92019-01-11 11:11:10 +0100782 if (rate_allocator_ && last_observed_bitrate_bps_ > 0) {
783 // We have a new rate allocator instance and already configured target
784 // bitrate. Update the rate allocation and notify observsers.
Erik Språngd7329ca2019-02-21 21:19:53 +0100785 const uint32_t framerate_fps = GetInputFramerateFps();
786 SetEncoderRates(GetBitrateAllocationAndNotifyObserver(
787 last_observed_bitrate_bps_, framerate_fps),
788 framerate_fps);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100789 }
ilnik35b7de42017-03-15 04:24:21 -0700790
Niels Möller213618e2018-07-24 09:29:58 +0200791 encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams);
Per512ecb32016-09-23 15:52:06 +0200792
perkjfa10b552016-10-02 23:45:26 -0700793 pending_encoder_reconfiguration_ = false;
Erik Språng08127a92016-11-16 16:41:30 +0100794
Pera48ddb72016-09-29 11:48:50 +0200795 sink_->OnEncoderConfigurationChanged(
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100796 std::move(streams), encoder_config_.content_type,
797 encoder_config_.min_transmit_bitrate_bps);
kthelgason876222f2016-11-29 01:44:11 -0800798
Niels Möller7dc26b72017-12-06 10:27:48 +0100799 // Get the current target framerate, ie the maximum framerate as specified by
800 // the current codec configuration, or any limit imposed by cpu adaption in
801 // maintain-resolution or balanced mode. This is used to make sure overuse
802 // detection doesn't needlessly trigger in low and/or variable framerate
803 // scenarios.
804 int target_framerate = std::min(
805 max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps);
806 overuse_detector_->OnTargetFramerateUpdated(target_framerate);
Niels Möller2d061182018-04-24 09:13:08 +0200807
Erik Språng7ca375c2019-02-06 16:20:17 +0100808 ConfigureQualityScaler(info);
kthelgason2bc68642017-02-07 07:02:22 -0800809}
810
Erik Språng7ca375c2019-02-06 16:20:17 +0100811void VideoStreamEncoder::ConfigureQualityScaler(
812 const VideoEncoder::EncoderInfo& encoder_info) {
kthelgason2bc68642017-02-07 07:02:22 -0800813 RTC_DCHECK_RUN_ON(&encoder_queue_);
Erik Språng7ca375c2019-02-06 16:20:17 +0100814 const auto scaling_settings = encoder_info.scaling_settings;
asapersson36e9eb42017-03-31 05:29:12 -0700815 const bool quality_scaling_allowed =
asapersson91914e22017-06-01 00:34:08 -0700816 IsResolutionScalingEnabled(degradation_preference_) &&
Niels Möller225c7872018-02-22 15:03:53 +0100817 scaling_settings.thresholds;
kthelgason3af6cc02017-03-22 00:25:28 -0700818
asapersson36e9eb42017-03-31 05:29:12 -0700819 if (quality_scaling_allowed) {
asapersson09f05612017-05-15 23:40:18 -0700820 if (quality_scaler_.get() == nullptr) {
821 // Quality scaler has not already been configured.
Niels Möller225c7872018-02-22 15:03:53 +0100822
Åsa Perssona945aee2018-04-24 16:53:25 +0200823 // Use experimental thresholds if available.
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200824 absl::optional<VideoEncoder::QpThresholds> experimental_thresholds;
Åsa Perssona945aee2018-04-24 16:53:25 +0200825 if (quality_scaling_experiment_enabled_) {
826 experimental_thresholds = QualityScalingExperiment::GetQpThresholds(
827 encoder_config_.codec_type);
828 }
Karl Wiberg918f50c2018-07-05 11:40:33 +0200829 // Since the interface is non-public, absl::make_unique can't do this
830 // upcast.
Niels Möller225c7872018-02-22 15:03:53 +0100831 AdaptationObserverInterface* observer = this;
Karl Wiberg918f50c2018-07-05 11:40:33 +0200832 quality_scaler_ = absl::make_unique<QualityScaler>(
Sebastian Janssoncda86dd2019-03-11 17:26:36 +0100833 &encoder_queue_, observer,
834 experimental_thresholds ? *experimental_thresholds
835 : *(scaling_settings.thresholds));
Kári Tristan Helgason639602a2018-08-02 10:51:40 +0200836 has_seen_first_significant_bwe_change_ = false;
837 initial_framedrop_ = 0;
kthelgason876222f2016-11-29 01:44:11 -0800838 }
839 } else {
840 quality_scaler_.reset(nullptr);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +0200841 initial_framedrop_ = kMaxInitialFramedrop;
kthelgason876222f2016-11-29 01:44:11 -0800842 }
asapersson09f05612017-05-15 23:40:18 -0700843
Niels Möller213618e2018-07-24 09:29:58 +0200844 encoder_stats_observer_->OnAdaptationChanged(
845 VideoStreamEncoderObserver::AdaptationReason::kNone,
846 GetActiveCounts(kCpu), GetActiveCounts(kQuality));
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000847}
848
mflodmancc3d4422017-08-03 08:27:51 -0700849void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {
perkj26091b12016-09-01 01:17:40 -0700850 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
perkj26091b12016-09-01 01:17:40 -0700851 VideoFrame incoming_frame = video_frame;
852
853 // Local time in webrtc time base.
ilnik04f4d122017-06-19 07:18:55 -0700854 int64_t current_time_us = clock_->TimeInMicroseconds();
855 int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec;
856 // In some cases, e.g., when the frame from decoder is fed to encoder,
857 // the timestamp may be set to the future. As the encoding pipeline assumes
858 // capture time to be less than present time, we should reset the capture
859 // timestamps here. Otherwise there may be issues with RTP send stream.
860 if (incoming_frame.timestamp_us() > current_time_us)
861 incoming_frame.set_timestamp_us(current_time_us);
perkj26091b12016-09-01 01:17:40 -0700862
863 // Capture time may come from clock with an offset and drift from clock_.
864 int64_t capture_ntp_time_ms;
nisse891419f2017-01-12 10:02:22 -0800865 if (video_frame.ntp_time_ms() > 0) {
perkj26091b12016-09-01 01:17:40 -0700866 capture_ntp_time_ms = video_frame.ntp_time_ms();
867 } else if (video_frame.render_time_ms() != 0) {
868 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
869 } else {
nisse1c0dea82017-01-30 02:43:18 -0800870 capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_;
perkj26091b12016-09-01 01:17:40 -0700871 }
872 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);
873
874 // Convert NTP time, in ms, to RTP timestamp.
875 const int kMsToRtpTimestamp = 90;
876 incoming_frame.set_timestamp(
877 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms()));
878
879 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
880 // We don't allow the same capture time for two frames, drop this one.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100881 RTC_LOG(LS_WARNING) << "Same/old NTP timestamp ("
882 << incoming_frame.ntp_time_ms()
883 << " <= " << last_captured_timestamp_
884 << ") for incoming frame. Dropping.";
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100885 encoder_queue_.PostTask([this, incoming_frame]() {
886 RTC_DCHECK_RUN_ON(&encoder_queue_);
887 accumulated_update_rect_.Union(incoming_frame.update_rect());
888 });
perkj26091b12016-09-01 01:17:40 -0700889 return;
890 }
891
asapersson6ffb67d2016-09-12 00:10:45 -0700892 bool log_stats = false;
nisse1c0dea82017-01-30 02:43:18 -0800893 if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) {
894 last_frame_log_ms_ = current_time_ms;
asapersson6ffb67d2016-09-12 00:10:45 -0700895 log_stats = true;
896 }
897
perkj26091b12016-09-01 01:17:40 -0700898 last_captured_timestamp_ = incoming_frame.ntp_time_ms();
Sebastian Jansson3ab5c402018-04-05 12:30:50 +0200899
900 int64_t post_time_us = rtc::TimeMicros();
901 ++posted_frames_waiting_for_encode_;
902
903 encoder_queue_.PostTask(
904 [this, incoming_frame, post_time_us, log_stats]() {
905 RTC_DCHECK_RUN_ON(&encoder_queue_);
Niels Möller213618e2018-07-24 09:29:58 +0200906 encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(),
907 incoming_frame.height());
Sebastian Jansson3ab5c402018-04-05 12:30:50 +0200908 ++captured_frame_count_;
909 const int posted_frames_waiting_for_encode =
910 posted_frames_waiting_for_encode_.fetch_sub(1);
911 RTC_DCHECK_GT(posted_frames_waiting_for_encode, 0);
912 if (posted_frames_waiting_for_encode == 1) {
Sebastian Janssona3177052018-04-10 13:05:49 +0200913 MaybeEncodeVideoFrame(incoming_frame, post_time_us);
Sebastian Jansson3ab5c402018-04-05 12:30:50 +0200914 } else {
915 // There is a newer frame in flight. Do not encode this frame.
916 RTC_LOG(LS_VERBOSE)
917 << "Incoming frame dropped due to that the encoder is blocked.";
918 ++dropped_frame_count_;
Niels Möller213618e2018-07-24 09:29:58 +0200919 encoder_stats_observer_->OnFrameDropped(
920 VideoStreamEncoderObserver::DropReason::kEncoderQueue);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100921 accumulated_update_rect_.Union(incoming_frame.update_rect());
Sebastian Jansson3ab5c402018-04-05 12:30:50 +0200922 }
923 if (log_stats) {
924 RTC_LOG(LS_INFO) << "Number of frames: captured "
925 << captured_frame_count_
926 << ", dropped (due to encoder blocked) "
927 << dropped_frame_count_ << ", interval_ms "
928 << kFrameLogIntervalMs;
929 captured_frame_count_ = 0;
930 dropped_frame_count_ = 0;
931 }
932 });
perkj26091b12016-09-01 01:17:40 -0700933}
934
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200935void VideoStreamEncoder::OnDiscardedFrame() {
Niels Möller213618e2018-07-24 09:29:58 +0200936 encoder_stats_observer_->OnFrameDropped(
937 VideoStreamEncoderObserver::DropReason::kSource);
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200938}
939
mflodmancc3d4422017-08-03 08:27:51 -0700940bool VideoStreamEncoder::EncoderPaused() const {
perkj26091b12016-09-01 01:17:40 -0700941 RTC_DCHECK_RUN_ON(&encoder_queue_);
pwestin@webrtc.org91563e42013-04-25 22:20:08 +0000942 // Pause video if paused by caller or as long as the network is down or the
943 // pacer queue has grown too large in buffered mode.
perkj57c21f92016-06-17 07:27:16 -0700944 // If the pacer queue has grown too large or the network is down,
perkjfea93092016-05-14 00:58:48 -0700945 // last_observed_bitrate_bps_ will be 0.
perkj26091b12016-09-01 01:17:40 -0700946 return last_observed_bitrate_bps_ == 0;
stefan@webrtc.orgbfacda62013-03-27 16:36:01 +0000947}
948
mflodmancc3d4422017-08-03 08:27:51 -0700949void VideoStreamEncoder::TraceFrameDropStart() {
perkj26091b12016-09-01 01:17:40 -0700950 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000951 // Start trace event only on the first frame after encoder is paused.
952 if (!encoder_paused_and_dropped_frame_) {
953 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this);
954 }
955 encoder_paused_and_dropped_frame_ = true;
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000956}
957
mflodmancc3d4422017-08-03 08:27:51 -0700958void VideoStreamEncoder::TraceFrameDropEnd() {
perkj26091b12016-09-01 01:17:40 -0700959 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000960 // End trace event on first frame after encoder resumes, if frame was dropped.
961 if (encoder_paused_and_dropped_frame_) {
962 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
963 }
964 encoder_paused_and_dropped_frame_ = false;
965}
966
Niels Möller6bb5ab92019-01-11 11:11:10 +0100967VideoBitrateAllocation
968VideoStreamEncoder::GetBitrateAllocationAndNotifyObserver(
969 const uint32_t target_bitrate_bps,
970 uint32_t framerate_fps) {
971 // Only call allocators if bitrate > 0 (ie, not suspended), otherwise they
972 // might cap the bitrate to the min bitrate configured.
973 VideoBitrateAllocation bitrate_allocation;
974 if (rate_allocator_ && target_bitrate_bps > 0) {
975 bitrate_allocation =
976 rate_allocator_->GetAllocation(target_bitrate_bps, framerate_fps);
977 }
978
979 if (bitrate_observer_ && bitrate_allocation.get_sum_bps() > 0) {
980 bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation);
981 }
982
Erik Språng7ca375c2019-02-06 16:20:17 +0100983 if (bitrate_adjuster_) {
Erik Språng0e1a1f92019-02-18 18:45:13 +0100984 VideoBitrateAllocation adjusted_allocation =
985 bitrate_adjuster_->AdjustRateAllocation(bitrate_allocation,
986 framerate_fps);
987 RTC_LOG(LS_VERBOSE) << "Adjusting allocation, fps = " << framerate_fps
988 << ", from " << bitrate_allocation.ToString() << ", to "
989 << adjusted_allocation.ToString();
990 return adjusted_allocation;
Erik Språng7ca375c2019-02-06 16:20:17 +0100991 }
Niels Möller6bb5ab92019-01-11 11:11:10 +0100992 return bitrate_allocation;
993}
994
995uint32_t VideoStreamEncoder::GetInputFramerateFps() {
996 const uint32_t default_fps = max_framerate_ != -1 ? max_framerate_ : 30;
Erik Språngd7329ca2019-02-21 21:19:53 +0100997 absl::optional<uint32_t> input_fps =
998 input_framerate_.Rate(clock_->TimeInMilliseconds());
999 if (!input_fps || *input_fps == 0) {
1000 return default_fps;
1001 }
1002 return *input_fps;
1003}
1004
1005void VideoStreamEncoder::SetEncoderRates(
1006 const VideoBitrateAllocation& bitrate_allocation,
1007 uint32_t framerate_fps) {
Erik Språng6a7baa72019-02-26 18:31:00 +01001008 if (!encoder_) {
Erik Språngd7329ca2019-02-21 21:19:53 +01001009 return;
1010 }
1011
1012 // |bitrate_allocation| is 0 it means that the network is down or the send
1013 // pacer is full. We currently only report this if the encoder has an internal
1014 // source. If the encoder does not have an internal source, higher levels
1015 // are expected to not call AddVideoFrame. We do this since its unclear
1016 // how current encoder implementations behave when given a zero target
1017 // bitrate.
1018 // TODO(perkj): Make sure all known encoder implementations handle zero
1019 // target bitrate and remove this check.
1020 if (!HasInternalSource() && bitrate_allocation.get_sum_bps() == 0) {
1021 return;
1022 }
1023
1024 RTC_DCHECK_GT(framerate_fps, 0);
Erik Språng6a7baa72019-02-26 18:31:00 +01001025 if (bitrate_allocation != last_bitrate_allocation_ ||
1026 framerate_fps != last_framerate_fps_) {
1027 int res = encoder_->SetRateAllocation(bitrate_allocation, framerate_fps);
1028 if (res != 0) {
1029 RTC_LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
1030 << bitrate_allocation.get_sum_bps()
1031 << ", framerate = " << framerate_fps << "): " << res;
1032 }
1033
1034 frame_encoder_timer_.OnSetRates(bitrate_allocation, framerate_fps);
1035 }
1036
1037 last_bitrate_allocation_ = bitrate_allocation;
1038 last_framerate_fps_ = framerate_fps;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001039}
1040
Sebastian Janssona3177052018-04-10 13:05:49 +02001041void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
1042 int64_t time_when_posted_us) {
perkj26091b12016-09-01 01:17:40 -07001043 RTC_DCHECK_RUN_ON(&encoder_queue_);
kthelgason876222f2016-11-29 01:44:11 -08001044
Per21d45d22016-10-30 21:37:57 +01001045 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
perkjfa10b552016-10-02 23:45:26 -07001046 video_frame.height() != last_frame_info_->height ||
perkjfa10b552016-10-02 23:45:26 -07001047 video_frame.is_texture() != last_frame_info_->is_texture) {
1048 pending_encoder_reconfiguration_ = true;
Oskar Sundbom8e07c132018-01-08 16:45:42 +01001049 last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(),
1050 video_frame.is_texture());
Mirko Bonadei675513b2017-11-09 11:09:25 +01001051 RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions="
1052 << last_frame_info_->width << "x"
1053 << last_frame_info_->height
1054 << ", texture=" << last_frame_info_->is_texture << ".";
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001055 // Force full frame update, since resolution has changed.
1056 accumulated_update_rect_ =
1057 VideoFrame::UpdateRect{0, 0, video_frame.width(), video_frame.height()};
perkjfa10b552016-10-02 23:45:26 -07001058 }
1059
Niels Möller4db138e2018-04-19 09:04:13 +02001060 // We have to create then encoder before the frame drop logic,
1061 // because the latter depends on encoder_->GetScalingSettings.
1062 // According to the testcase
1063 // InitialFrameDropOffWhenEncoderDisabledScaling, the return value
1064 // from GetScalingSettings should enable or disable the frame drop.
1065
Erik Språnga8d48ab2019-02-08 14:17:40 +01001066 // Update input frame rate before we start using it. If we update it after
Erik Språngd7329ca2019-02-21 21:19:53 +01001067 // any potential frame drop we are going to artificially increase frame sizes.
1068 // Poll the rate before updating, otherwise we risk the rate being estimated
1069 // a little too high at the start of the call when then window is small.
Niels Möller6bb5ab92019-01-11 11:11:10 +01001070 uint32_t framerate_fps = GetInputFramerateFps();
Erik Språngd7329ca2019-02-21 21:19:53 +01001071 input_framerate_.Update(1u, clock_->TimeInMilliseconds());
Niels Möller6bb5ab92019-01-11 11:11:10 +01001072
Niels Möller4db138e2018-04-19 09:04:13 +02001073 int64_t now_ms = clock_->TimeInMilliseconds();
1074 if (pending_encoder_reconfiguration_) {
1075 ReconfigureEncoder();
1076 last_parameters_update_ms_.emplace(now_ms);
1077 } else if (!last_parameters_update_ms_ ||
1078 now_ms - *last_parameters_update_ms_ >=
1079 vcm::VCMProcessTimer::kDefaultProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01001080 SetEncoderRates(GetBitrateAllocationAndNotifyObserver(
1081 last_observed_bitrate_bps_, framerate_fps),
1082 framerate_fps);
Niels Möller4db138e2018-04-19 09:04:13 +02001083 last_parameters_update_ms_.emplace(now_ms);
1084 }
1085
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001086 // Because pending frame will be dropped in any case, we need to
1087 // remember its updated region.
1088 if (pending_frame_) {
1089 encoder_stats_observer_->OnFrameDropped(
1090 VideoStreamEncoderObserver::DropReason::kEncoderQueue);
1091 accumulated_update_rect_.Union(pending_frame_->update_rect());
1092 }
1093
Sebastian Janssona3177052018-04-10 13:05:49 +02001094 if (DropDueToSize(video_frame.size())) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001095 RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
Åsa Persson875841d2018-01-08 08:49:53 +01001096 int count = GetConstAdaptCounter().ResolutionCount(kQuality);
kthelgason2bc68642017-02-07 07:02:22 -08001097 AdaptDown(kQuality);
Åsa Persson875841d2018-01-08 08:49:53 +01001098 if (GetConstAdaptCounter().ResolutionCount(kQuality) > count) {
Niels Möller213618e2018-07-24 09:29:58 +02001099 encoder_stats_observer_->OnInitialQualityResolutionAdaptDown();
Åsa Persson875841d2018-01-08 08:49:53 +01001100 }
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02001101 ++initial_framedrop_;
Sebastian Jansson0d70e372018-04-17 13:57:13 +02001102 // Storing references to a native buffer risks blocking frame capture.
1103 if (video_frame.video_frame_buffer()->type() !=
1104 VideoFrameBuffer::Type::kNative) {
1105 pending_frame_ = video_frame;
1106 pending_frame_post_time_us_ = time_when_posted_us;
1107 } else {
1108 // Ensure that any previously stored frame is dropped.
1109 pending_frame_.reset();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001110 accumulated_update_rect_.Union(video_frame.update_rect());
Sebastian Jansson0d70e372018-04-17 13:57:13 +02001111 }
kthelgason2bc68642017-02-07 07:02:22 -08001112 return;
1113 }
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02001114 initial_framedrop_ = kMaxInitialFramedrop;
kthelgason2bc68642017-02-07 07:02:22 -08001115
perkj26091b12016-09-01 01:17:40 -07001116 if (EncoderPaused()) {
Sebastian Jansson0d70e372018-04-17 13:57:13 +02001117 // Storing references to a native buffer risks blocking frame capture.
1118 if (video_frame.video_frame_buffer()->type() !=
1119 VideoFrameBuffer::Type::kNative) {
1120 if (pending_frame_)
1121 TraceFrameDropStart();
1122 pending_frame_ = video_frame;
1123 pending_frame_post_time_us_ = time_when_posted_us;
1124 } else {
1125 // Ensure that any previously stored frame is dropped.
1126 pending_frame_.reset();
Sebastian Janssona3177052018-04-10 13:05:49 +02001127 TraceFrameDropStart();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001128 accumulated_update_rect_.Union(video_frame.update_rect());
Sebastian Jansson0d70e372018-04-17 13:57:13 +02001129 }
perkj26091b12016-09-01 01:17:40 -07001130 return;
mflodman@webrtc.org84d17832011-12-01 17:02:23 +00001131 }
Sebastian Janssona3177052018-04-10 13:05:49 +02001132
1133 pending_frame_.reset();
Niels Möller6bb5ab92019-01-11 11:11:10 +01001134
1135 frame_dropper_.Leak(framerate_fps);
1136 // Frame dropping is enabled iff frame dropping is not force-disabled, and
1137 // rate controller is not trusted.
1138 const bool frame_dropping_enabled =
1139 !force_disable_frame_dropper_ &&
1140 !encoder_info_.has_trusted_rate_controller;
1141 frame_dropper_.Enable(frame_dropping_enabled);
1142 if (frame_dropping_enabled && frame_dropper_.DropFrame()) {
1143 RTC_LOG(LS_VERBOSE) << "Drop Frame: "
1144 << "target bitrate " << last_observed_bitrate_bps_
1145 << ", input frame rate " << framerate_fps;
1146 OnDroppedFrame(
1147 EncodedImageCallback::DropReason::kDroppedByMediaOptimizations);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001148 accumulated_update_rect_.Union(video_frame.update_rect());
Niels Möller6bb5ab92019-01-11 11:11:10 +01001149 return;
1150 }
1151
Sebastian Janssona3177052018-04-10 13:05:49 +02001152 EncodeVideoFrame(video_frame, time_when_posted_us);
1153}
1154
1155void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
1156 int64_t time_when_posted_us) {
1157 RTC_DCHECK_RUN_ON(&encoder_queue_);
perkj26091b12016-09-01 01:17:40 -07001158 TraceFrameDropEnd();
niklase@google.com470e71d2011-07-07 08:21:25 +00001159
ilnik6b826ef2017-06-16 06:53:48 -07001160 VideoFrame out_frame(video_frame);
1161 // Crop frame if needed.
1162 if (crop_width_ > 0 || crop_height_ > 0) {
1163 int cropped_width = video_frame.width() - crop_width_;
1164 int cropped_height = video_frame.height() - crop_height_;
1165 rtc::scoped_refptr<I420Buffer> cropped_buffer =
1166 I420Buffer::Create(cropped_width, cropped_height);
1167 // TODO(ilnik): Remove scaling if cropping is too big, as it should never
1168 // happen after SinkWants signaled correctly from ReconfigureEncoder.
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001169 VideoFrame::UpdateRect update_rect = video_frame.update_rect();
ilnik6b826ef2017-06-16 06:53:48 -07001170 if (crop_width_ < 4 && crop_height_ < 4) {
1171 cropped_buffer->CropAndScaleFrom(
1172 *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2,
1173 crop_height_ / 2, cropped_width, cropped_height);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001174 update_rect.offset_x -= crop_width_ / 2;
1175 update_rect.offset_y -= crop_height_ / 2;
1176 update_rect.Intersect(
1177 VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height});
1178
ilnik6b826ef2017-06-16 06:53:48 -07001179 } else {
1180 cropped_buffer->ScaleFrom(
1181 *video_frame.video_frame_buffer()->ToI420().get());
Ilya Nikolaevskiy1c90cab2019-03-07 15:30:58 +01001182 if (!update_rect.IsEmpty()) {
1183 // Since we can't reason about pixels after scaling, we invalidate whole
1184 // picture, if anything changed.
1185 update_rect =
1186 VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height};
1187 }
ilnik6b826ef2017-06-16 06:53:48 -07001188 }
Artem Titov1ebfb6a2019-01-03 23:49:37 +01001189 out_frame = VideoFrame::Builder()
1190 .set_video_frame_buffer(cropped_buffer)
1191 .set_timestamp_rtp(video_frame.timestamp())
1192 .set_timestamp_ms(video_frame.render_time_ms())
1193 .set_rotation(video_frame.rotation())
1194 .set_id(video_frame.id())
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001195 .set_update_rect(update_rect)
Artem Titov1ebfb6a2019-01-03 23:49:37 +01001196 .build();
ilnik6b826ef2017-06-16 06:53:48 -07001197 out_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001198 // Since accumulated_update_rect_ is constructed before cropping,
1199 // we can't trust it. If any changes were pending, we invalidate whole
1200 // frame here.
1201 if (!accumulated_update_rect_.IsEmpty()) {
1202 accumulated_update_rect_ =
1203 VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()};
1204 }
1205 }
1206
1207 if (!accumulated_update_rect_.IsEmpty()) {
1208 accumulated_update_rect_.Union(out_frame.update_rect());
1209 accumulated_update_rect_.Intersect(
1210 VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()});
1211 out_frame.set_update_rect(accumulated_update_rect_);
1212 accumulated_update_rect_.MakeEmptyUpdate();
ilnik6b826ef2017-06-16 06:53:48 -07001213 }
1214
Magnus Jedvert26679d62015-04-07 14:07:41 +02001215 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
hclam@chromium.org1a7b9b92013-07-08 21:31:18 +00001216 "Encode");
pbos@webrtc.orgfe1ef932013-10-21 10:34:43 +00001217
Niels Möller7dc26b72017-12-06 10:27:48 +01001218 overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);
perkjd52063f2016-09-07 06:32:18 -07001219
Erik Språnge2fd86a2018-10-24 11:32:39 +02001220 // Encoder metadata needs to be updated before encode complete callback.
1221 VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo();
1222 if (info.implementation_name != encoder_info_.implementation_name) {
1223 encoder_stats_observer_->OnEncoderImplementationChanged(
1224 info.implementation_name);
Erik Språng7ca375c2019-02-06 16:20:17 +01001225 if (bitrate_adjuster_) {
1226 // Encoder implementation changed, reset overshoot detector states.
1227 bitrate_adjuster_->Reset();
1228 }
Erik Språnge2fd86a2018-10-24 11:32:39 +02001229 }
Erik Språng7ca375c2019-02-06 16:20:17 +01001230
1231 if (bitrate_adjuster_) {
1232 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1233 if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) {
1234 bitrate_adjuster_->OnEncoderInfo(info);
1235 break;
1236 }
1237 }
1238 }
1239
Erik Språnge2fd86a2018-10-24 11:32:39 +02001240 encoder_info_ = info;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001241 RTC_DCHECK_EQ(send_codec_.width, out_frame.width());
1242 RTC_DCHECK_EQ(send_codec_.height, out_frame.height());
Erik Språngd7329ca2019-02-21 21:19:53 +01001243 const VideoFrameBuffer::Type buffer_type =
1244 out_frame.video_frame_buffer()->type();
1245 const bool is_buffer_type_supported =
1246 buffer_type == VideoFrameBuffer::Type::kI420 ||
1247 (buffer_type == VideoFrameBuffer::Type::kNative &&
Erik Språng6a7baa72019-02-26 18:31:00 +01001248 info.supports_native_handle);
Erik Språngd7329ca2019-02-21 21:19:53 +01001249
1250 if (!is_buffer_type_supported) {
1251 // This module only supports software encoding.
1252 rtc::scoped_refptr<I420BufferInterface> converted_buffer(
1253 out_frame.video_frame_buffer()->ToI420());
1254
1255 if (!converted_buffer) {
1256 RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame.";
1257 return;
1258 }
1259
Ilya Nikolaevskiy1c90cab2019-03-07 15:30:58 +01001260 // UpdatedRect is reset to full update if it's not empty, because buffer was
1261 // converted, therefore we can't guarantee that pixels outside of UpdateRect
1262 // didn't change comparing to the previous frame.
1263 VideoFrame::UpdateRect update_rect =
1264 out_frame.update_rect().IsEmpty()
1265 ? out_frame.update_rect()
1266 : VideoFrame::UpdateRect{0, 0, out_frame.width(),
1267 out_frame.height()};
1268
Erik Språngd7329ca2019-02-21 21:19:53 +01001269 out_frame = VideoFrame::Builder()
1270 .set_video_frame_buffer(converted_buffer)
1271 .set_timestamp_rtp(out_frame.timestamp())
1272 .set_timestamp_ms(out_frame.render_time_ms())
1273 .set_rotation(out_frame.rotation())
1274 .set_id(out_frame.id())
Ilya Nikolaevskiy1c90cab2019-03-07 15:30:58 +01001275 .set_update_rect(update_rect)
Erik Språngd7329ca2019-02-21 21:19:53 +01001276 .build();
1277 }
Erik Språng6a7baa72019-02-26 18:31:00 +01001278
1279 TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
1280 out_frame.timestamp());
1281
1282 frame_encoder_timer_.OnEncodeStarted(out_frame.timestamp(),
1283 out_frame.render_time_ms());
1284
Niels Möllerc8d2e732019-03-06 12:00:33 +01001285 const int32_t encode_status = encoder_->Encode(out_frame, &next_frame_types_);
Erik Språng6a7baa72019-02-26 18:31:00 +01001286
Erik Språngd7329ca2019-02-21 21:19:53 +01001287 if (encode_status < 0) {
1288 RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: "
1289 << encode_status;
1290 return;
1291 }
1292
1293 for (auto& it : next_frame_types_) {
1294 it = kVideoFrameDelta;
1295 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001296}
niklase@google.com470e71d2011-07-07 08:21:25 +00001297
mflodmancc3d4422017-08-03 08:27:51 -07001298void VideoStreamEncoder::SendKeyFrame() {
perkj26091b12016-09-01 01:17:40 -07001299 if (!encoder_queue_.IsCurrent()) {
1300 encoder_queue_.PostTask([this] { SendKeyFrame(); });
1301 return;
1302 }
1303 RTC_DCHECK_RUN_ON(&encoder_queue_);
Niels Möller1c9aa1e2018-02-16 10:27:23 +01001304 TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
Erik Språngd7329ca2019-02-21 21:19:53 +01001305 RTC_DCHECK(!next_frame_types_.empty());
1306 next_frame_types_[0] = kVideoFrameKey;
1307 if (HasInternalSource()) {
1308 // Try to request the frame if we have an external encoder with
1309 // internal source since AddVideoFrame never will be called.
Erik Språng6a7baa72019-02-26 18:31:00 +01001310
1311 // TODO(nisse): Used only with internal source. Delete as soon as
1312 // that feature is removed. The only implementation I've been able
1313 // to find ignores what's in the frame. With one exception: It seems
1314 // a few test cases, e.g.,
1315 // VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set
1316 // internal_source to true and use FakeEncoder. And the latter will
1317 // happily encode this 1x1 frame and pass it on down the pipeline.
1318 if (encoder_->Encode(VideoFrame::Builder()
1319 .set_video_frame_buffer(I420Buffer::Create(1, 1))
1320 .set_rotation(kVideoRotation_0)
1321 .set_timestamp_us(0)
1322 .build(),
Erik Språng6a7baa72019-02-26 18:31:00 +01001323 &next_frame_types_) == WEBRTC_VIDEO_CODEC_OK) {
Erik Språngd7329ca2019-02-21 21:19:53 +01001324 // Try to remove just-performed keyframe request, if stream still exists.
1325 next_frame_types_[0] = kVideoFrameDelta;
1326 }
1327 }
stefan@webrtc.org07b45a52012-02-02 08:37:48 +00001328}
1329
mflodmancc3d4422017-08-03 08:27:51 -07001330EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage(
Sergey Ulanov525df3f2016-08-02 17:46:41 -07001331 const EncodedImage& encoded_image,
1332 const CodecSpecificInfo* codec_specific_info,
1333 const RTPFragmentationHeader* fragmentation) {
Erik Språng6a7baa72019-02-26 18:31:00 +01001334 const int64_t time_sent_us = rtc::TimeMicros();
1335
1336 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
1337 "timestamp", encoded_image.Timestamp());
1338 const size_t spatial_idx = encoded_image.SpatialIndex().value_or(0);
1339 EncodedImage image_copy(encoded_image);
1340
1341 frame_encoder_timer_.FillTimingInfo(
1342 spatial_idx, &image_copy, time_sent_us / rtc::kNumMicrosecsPerMillisec);
1343
1344 // Piggyback ALR experiment group id and simulcast id into the content type.
1345 const uint8_t experiment_id =
1346 experiment_groups_[videocontenttypehelpers::IsScreenshare(
1347 image_copy.content_type_)];
1348
1349 // TODO(ilnik): This will force content type extension to be present even
1350 // for realtime video. At the expense of miniscule overhead we will get
1351 // sliced receive statistics.
1352 RTC_CHECK(videocontenttypehelpers::SetExperimentId(&image_copy.content_type_,
1353 experiment_id));
1354 // We count simulcast streams from 1 on the wire. That's why we set simulcast
1355 // id in content type to +1 of that is actual simulcast index. This is because
1356 // value 0 on the wire is reserved for 'no simulcast stream specified'.
1357 RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
1358 &image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1)));
1359
perkj26091b12016-09-01 01:17:40 -07001360 // Encoded is called on whatever thread the real encoder implementation run
1361 // on. In the case of hardware encoders, there might be several encoders
1362 // running in parallel on different threads.
Erik Språng6a7baa72019-02-26 18:31:00 +01001363 encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info);
sprang3911c262016-04-15 01:24:14 -07001364
Sergey Ulanov525df3f2016-08-02 17:46:41 -07001365 EncodedImageCallback::Result result =
Erik Språng6a7baa72019-02-26 18:31:00 +01001366 sink_->OnEncodedImage(image_copy, codec_specific_info, fragmentation);
perkjbc75d972016-05-02 06:31:25 -07001367
Erik Språng7ca375c2019-02-06 16:20:17 +01001368 // We are only interested in propagating the meta-data about the image, not
1369 // encoded data itself, to the post encode function. Since we cannot be sure
1370 // the pointer will still be valid when run on the task queue, set it to null.
Erik Språng6a7baa72019-02-26 18:31:00 +01001371 image_copy.set_buffer(nullptr, 0);
Niels Möller83dbeac2017-12-14 16:39:44 +01001372
Erik Språng7ca375c2019-02-06 16:20:17 +01001373 int temporal_index = 0;
1374 if (codec_specific_info) {
1375 if (codec_specific_info->codecType == kVideoCodecVP9) {
1376 temporal_index = codec_specific_info->codecSpecific.VP9.temporal_idx;
1377 } else if (codec_specific_info->codecType == kVideoCodecVP8) {
1378 temporal_index = codec_specific_info->codecSpecific.VP8.temporalIdx;
1379 }
1380 }
1381 if (temporal_index == kNoTemporalIdx) {
1382 temporal_index = 0;
Niels Möller83dbeac2017-12-14 16:39:44 +01001383 }
1384
Erik Språng6a7baa72019-02-26 18:31:00 +01001385 RunPostEncode(image_copy, time_sent_us, temporal_index);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001386
1387 if (result.error == Result::OK) {
1388 // In case of an internal encoder running on a separate thread, the
1389 // decision to drop a frame might be a frame late and signaled via
1390 // atomic flag. This is because we can't easily wait for the worker thread
1391 // without risking deadlocks, eg during shutdown when the worker thread
1392 // might be waiting for the internal encoder threads to stop.
1393 if (pending_frame_drops_.load() > 0) {
1394 int pending_drops = pending_frame_drops_.fetch_sub(1);
1395 RTC_DCHECK_GT(pending_drops, 0);
1396 result.drop_next_frame = true;
1397 }
1398 }
perkj803d97f2016-11-01 11:45:46 -07001399
Sergey Ulanov525df3f2016-08-02 17:46:41 -07001400 return result;
Peter Boströmb7d9a972015-12-18 16:01:11 +01001401}
1402
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +02001403void VideoStreamEncoder::OnDroppedFrame(DropReason reason) {
1404 switch (reason) {
1405 case DropReason::kDroppedByMediaOptimizations:
Niels Möller213618e2018-07-24 09:29:58 +02001406 encoder_stats_observer_->OnFrameDropped(
1407 VideoStreamEncoderObserver::DropReason::kMediaOptimization);
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +02001408 encoder_queue_.PostTask([this] {
1409 RTC_DCHECK_RUN_ON(&encoder_queue_);
1410 if (quality_scaler_)
Åsa Perssona945aee2018-04-24 16:53:25 +02001411 quality_scaler_->ReportDroppedFrameByMediaOpt();
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +02001412 });
1413 break;
1414 case DropReason::kDroppedByEncoder:
Niels Möller213618e2018-07-24 09:29:58 +02001415 encoder_stats_observer_->OnFrameDropped(
1416 VideoStreamEncoderObserver::DropReason::kEncoder);
Åsa Perssona945aee2018-04-24 16:53:25 +02001417 encoder_queue_.PostTask([this] {
1418 RTC_DCHECK_RUN_ON(&encoder_queue_);
1419 if (quality_scaler_)
1420 quality_scaler_->ReportDroppedFrameByEncoder();
1421 });
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +02001422 break;
1423 }
kthelgason876222f2016-11-29 01:44:11 -08001424}
1425
Erik Språng610c7632019-03-06 15:37:33 +01001426void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate,
1427 DataRate target_headroom,
mflodmancc3d4422017-08-03 08:27:51 -07001428 uint8_t fraction_lost,
1429 int64_t round_trip_time_ms) {
perkj26091b12016-09-01 01:17:40 -07001430 if (!encoder_queue_.IsCurrent()) {
Erik Språng610c7632019-03-06 15:37:33 +01001431 encoder_queue_.PostTask([this, target_bitrate, target_headroom,
1432 fraction_lost, round_trip_time_ms] {
1433 OnBitrateUpdated(target_bitrate, target_headroom, fraction_lost,
1434 round_trip_time_ms);
1435 });
perkj26091b12016-09-01 01:17:40 -07001436 return;
1437 }
1438 RTC_DCHECK_RUN_ON(&encoder_queue_);
1439 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active.";
1440
Erik Språng610c7632019-03-06 15:37:33 +01001441 RTC_LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << target_bitrate.bps()
1442 << " headroom = " << target_headroom.bps()
Mirko Bonadei675513b2017-11-09 11:09:25 +01001443 << " packet loss " << static_cast<int>(fraction_lost)
1444 << " rtt " << round_trip_time_ms;
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02001445 // On significant changes to BWE at the start of the call,
1446 // enable frame drops to quickly react to jumps in available bandwidth.
1447 if (encoder_start_bitrate_bps_ != 0 &&
1448 !has_seen_first_significant_bwe_change_ && quality_scaler_ &&
1449 initial_framedrop_on_bwe_enabled_ &&
Erik Språng610c7632019-03-06 15:37:33 +01001450 abs_diff(target_bitrate.bps(), encoder_start_bitrate_bps_) >=
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02001451 kFramedropThreshold * encoder_start_bitrate_bps_) {
1452 // Reset initial framedrop feature when first real BW estimate arrives.
1453 // TODO(kthelgason): Update BitrateAllocator to not call OnBitrateUpdated
1454 // without an actual BW estimate.
1455 initial_framedrop_ = 0;
1456 has_seen_first_significant_bwe_change_ = true;
1457 }
perkj26091b12016-09-01 01:17:40 -07001458
Niels Möller6bb5ab92019-01-11 11:11:10 +01001459 uint32_t framerate_fps = GetInputFramerateFps();
Erik Språng610c7632019-03-06 15:37:33 +01001460 frame_dropper_.SetRates((target_bitrate.bps() + 500) / 1000, framerate_fps);
1461 SetEncoderRates(GetBitrateAllocationAndNotifyObserver(target_bitrate.bps(),
1462 framerate_fps),
1463 framerate_fps);
perkj26091b12016-09-01 01:17:40 -07001464
Erik Språng610c7632019-03-06 15:37:33 +01001465 encoder_start_bitrate_bps_ = target_bitrate.bps() != 0
1466 ? target_bitrate.bps()
1467 : encoder_start_bitrate_bps_;
1468 bool video_is_suspended = target_bitrate == DataRate::Zero();
Erik Språng08127a92016-11-16 16:41:30 +01001469 bool video_suspension_changed = video_is_suspended != EncoderPaused();
Erik Språng610c7632019-03-06 15:37:33 +01001470 last_observed_bitrate_bps_ = target_bitrate.bps();
Peter Boströmd153a372015-11-10 15:27:12 +00001471
sprang552c7c72017-02-13 04:41:45 -08001472 if (video_suspension_changed) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001473 RTC_LOG(LS_INFO) << "Video suspend state changed to: "
1474 << (video_is_suspended ? "suspended" : "not suspended");
Niels Möller213618e2018-07-24 09:29:58 +02001475 encoder_stats_observer_->OnSuspendChange(video_is_suspended);
mflodman101f2502016-06-09 17:21:19 +02001476 }
Sebastian Janssona3177052018-04-10 13:05:49 +02001477 if (video_suspension_changed && !video_is_suspended && pending_frame_ &&
1478 !DropDueToSize(pending_frame_->size())) {
1479 int64_t pending_time_us = rtc::TimeMicros() - pending_frame_post_time_us_;
1480 if (pending_time_us < kPendingFrameTimeoutMs * 1000)
1481 EncodeVideoFrame(*pending_frame_, pending_frame_post_time_us_);
1482 pending_frame_.reset();
1483 }
1484}
1485
1486bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02001487 if (initial_framedrop_ < kMaxInitialFramedrop &&
Sebastian Janssona3177052018-04-10 13:05:49 +02001488 encoder_start_bitrate_bps_ > 0) {
1489 if (encoder_start_bitrate_bps_ < 300000 /* qvga */) {
1490 return pixel_count > 320 * 240;
1491 } else if (encoder_start_bitrate_bps_ < 500000 /* vga */) {
1492 return pixel_count > 640 * 480;
1493 }
1494 }
1495 return false;
niklase@google.com470e71d2011-07-07 08:21:25 +00001496}
1497
mflodmancc3d4422017-08-03 08:27:51 -07001498void VideoStreamEncoder::AdaptDown(AdaptReason reason) {
perkjd52063f2016-09-07 06:32:18 -07001499 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangc5d62e22017-04-02 23:53:04 -07001500 AdaptationRequest adaptation_request = {
1501 last_frame_info_->pixel_count(),
Niels Möller213618e2018-07-24 09:29:58 +02001502 encoder_stats_observer_->GetInputFrameRate(),
sprangc5d62e22017-04-02 23:53:04 -07001503 AdaptationRequest::Mode::kAdaptDown};
asapersson09f05612017-05-15 23:40:18 -07001504
sprangc5d62e22017-04-02 23:53:04 -07001505 bool downgrade_requested =
1506 last_adaptation_request_ &&
1507 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
1508
sprangc5d62e22017-04-02 23:53:04 -07001509 switch (degradation_preference_) {
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001510 case DegradationPreference::BALANCED:
asaperssonf7e294d2017-06-13 23:25:22 -07001511 break;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001512 case DegradationPreference::MAINTAIN_FRAMERATE:
sprangc5d62e22017-04-02 23:53:04 -07001513 if (downgrade_requested &&
1514 adaptation_request.input_pixel_count_ >=
1515 last_adaptation_request_->input_pixel_count_) {
1516 // Don't request lower resolution if the current resolution is not
1517 // lower than the last time we asked for the resolution to be lowered.
1518 return;
1519 }
1520 break;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001521 case DegradationPreference::MAINTAIN_RESOLUTION:
sprangc5d62e22017-04-02 23:53:04 -07001522 if (adaptation_request.framerate_fps_ <= 0 ||
1523 (downgrade_requested &&
1524 adaptation_request.framerate_fps_ < kMinFramerateFps)) {
1525 // If no input fps estimate available, can't determine how to scale down
1526 // framerate. Otherwise, don't request lower framerate if we don't have
1527 // a valid frame rate. Since framerate, unlike resolution, is a measure
1528 // we have to estimate, and can fluctuate naturally over time, don't
1529 // make the same kind of limitations as for resolution, but trust the
1530 // overuse detector to not trigger too often.
1531 return;
1532 }
1533 break;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001534 case DegradationPreference::DISABLED:
sprangc5d62e22017-04-02 23:53:04 -07001535 return;
sprang84a37592017-02-10 07:04:27 -08001536 }
sprangc5d62e22017-04-02 23:53:04 -07001537
sprangc5d62e22017-04-02 23:53:04 -07001538 switch (degradation_preference_) {
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001539 case DegradationPreference::BALANCED: {
asaperssonf7e294d2017-06-13 23:25:22 -07001540 // Try scale down framerate, if lower.
1541 int fps = MinFps(last_frame_info_->pixel_count());
1542 if (source_proxy_->RestrictFramerate(fps)) {
1543 GetAdaptCounter().IncrementFramerate(reason);
1544 break;
1545 }
1546 // Scale down resolution.
Karl Wiberg80ba3332018-02-05 10:33:35 +01001547 RTC_FALLTHROUGH();
asaperssonf7e294d2017-06-13 23:25:22 -07001548 }
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001549 case DegradationPreference::MAINTAIN_FRAMERATE: {
asapersson13874762017-06-07 00:01:02 -07001550 // Scale down resolution.
Åsa Perssonc3ed6302017-11-16 14:04:52 +01001551 bool min_pixels_reached = false;
asaperssond0de2952017-04-21 01:47:31 -07001552 if (!source_proxy_->RequestResolutionLowerThan(
asapersson142fcc92017-08-17 08:58:54 -07001553 adaptation_request.input_pixel_count_,
Erik Språnge2fd86a2018-10-24 11:32:39 +02001554 encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame,
Åsa Perssonc3ed6302017-11-16 14:04:52 +01001555 &min_pixels_reached)) {
1556 if (min_pixels_reached)
Niels Möller213618e2018-07-24 09:29:58 +02001557 encoder_stats_observer_->OnMinPixelLimitReached();
asaperssond0de2952017-04-21 01:47:31 -07001558 return;
1559 }
asaperssonf7e294d2017-06-13 23:25:22 -07001560 GetAdaptCounter().IncrementResolution(reason);
sprangc5d62e22017-04-02 23:53:04 -07001561 break;
Åsa Perssonc3ed6302017-11-16 14:04:52 +01001562 }
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001563 case DegradationPreference::MAINTAIN_RESOLUTION: {
asapersson13874762017-06-07 00:01:02 -07001564 // Scale down framerate.
sprangfda496a2017-06-15 04:21:07 -07001565 const int requested_framerate = source_proxy_->RequestFramerateLowerThan(
1566 adaptation_request.framerate_fps_);
1567 if (requested_framerate == -1)
asapersson13874762017-06-07 00:01:02 -07001568 return;
sprangfda496a2017-06-15 04:21:07 -07001569 RTC_DCHECK_NE(max_framerate_, -1);
Niels Möller7dc26b72017-12-06 10:27:48 +01001570 overuse_detector_->OnTargetFramerateUpdated(
1571 std::min(max_framerate_, requested_framerate));
asaperssonf7e294d2017-06-13 23:25:22 -07001572 GetAdaptCounter().IncrementFramerate(reason);
sprangc5d62e22017-04-02 23:53:04 -07001573 break;
sprangfda496a2017-06-15 04:21:07 -07001574 }
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001575 case DegradationPreference::DISABLED:
sprangc5d62e22017-04-02 23:53:04 -07001576 RTC_NOTREACHED();
1577 }
1578
asaperssond0de2952017-04-21 01:47:31 -07001579 last_adaptation_request_.emplace(adaptation_request);
1580
asapersson09f05612017-05-15 23:40:18 -07001581 UpdateAdaptationStats(reason);
asaperssond0de2952017-04-21 01:47:31 -07001582
Mirko Bonadei675513b2017-11-09 11:09:25 +01001583 RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString();
perkj26091b12016-09-01 01:17:40 -07001584}
1585
mflodmancc3d4422017-08-03 08:27:51 -07001586void VideoStreamEncoder::AdaptUp(AdaptReason reason) {
perkjd52063f2016-09-07 06:32:18 -07001587 RTC_DCHECK_RUN_ON(&encoder_queue_);
asapersson09f05612017-05-15 23:40:18 -07001588
1589 const AdaptCounter& adapt_counter = GetConstAdaptCounter();
1590 int num_downgrades = adapt_counter.TotalCount(reason);
1591 if (num_downgrades == 0)
perkj803d97f2016-11-01 11:45:46 -07001592 return;
asapersson09f05612017-05-15 23:40:18 -07001593 RTC_DCHECK_GT(num_downgrades, 0);
1594
sprangc5d62e22017-04-02 23:53:04 -07001595 AdaptationRequest adaptation_request = {
1596 last_frame_info_->pixel_count(),
Niels Möller213618e2018-07-24 09:29:58 +02001597 encoder_stats_observer_->GetInputFrameRate(),
sprangc5d62e22017-04-02 23:53:04 -07001598 AdaptationRequest::Mode::kAdaptUp};
1599
1600 bool adapt_up_requested =
1601 last_adaptation_request_ &&
1602 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
asapersson09f05612017-05-15 23:40:18 -07001603
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001604 if (degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE) {
asaperssonf7e294d2017-06-13 23:25:22 -07001605 if (adapt_up_requested &&
1606 adaptation_request.input_pixel_count_ <=
1607 last_adaptation_request_->input_pixel_count_) {
1608 // Don't request higher resolution if the current resolution is not
1609 // higher than the last time we asked for the resolution to be higher.
sprangc5d62e22017-04-02 23:53:04 -07001610 return;
asaperssonf7e294d2017-06-13 23:25:22 -07001611 }
sprangb1ca0732017-02-01 08:38:12 -08001612 }
sprangc5d62e22017-04-02 23:53:04 -07001613
sprangc5d62e22017-04-02 23:53:04 -07001614 switch (degradation_preference_) {
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001615 case DegradationPreference::BALANCED: {
asaperssonf7e294d2017-06-13 23:25:22 -07001616 // Try scale up framerate, if higher.
1617 int fps = MaxFps(last_frame_info_->pixel_count());
1618 if (source_proxy_->IncreaseFramerate(fps)) {
1619 GetAdaptCounter().DecrementFramerate(reason, fps);
1620 // Reset framerate in case of fewer fps steps down than up.
1621 if (adapt_counter.FramerateCount() == 0 &&
1622 fps != std::numeric_limits<int>::max()) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001623 RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
asaperssonf7e294d2017-06-13 23:25:22 -07001624 source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max());
1625 }
1626 break;
1627 }
1628 // Scale up resolution.
Karl Wiberg80ba3332018-02-05 10:33:35 +01001629 RTC_FALLTHROUGH();
asaperssonf7e294d2017-06-13 23:25:22 -07001630 }
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001631 case DegradationPreference::MAINTAIN_FRAMERATE: {
asapersson13874762017-06-07 00:01:02 -07001632 // Scale up resolution.
1633 int pixel_count = adaptation_request.input_pixel_count_;
1634 if (adapt_counter.ResolutionCount() == 1) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001635 RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting.";
asapersson13874762017-06-07 00:01:02 -07001636 pixel_count = std::numeric_limits<int>::max();
sprangc5d62e22017-04-02 23:53:04 -07001637 }
asapersson13874762017-06-07 00:01:02 -07001638 if (!source_proxy_->RequestHigherResolutionThan(pixel_count))
1639 return;
asaperssonf7e294d2017-06-13 23:25:22 -07001640 GetAdaptCounter().DecrementResolution(reason);
sprangc5d62e22017-04-02 23:53:04 -07001641 break;
asapersson13874762017-06-07 00:01:02 -07001642 }
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001643 case DegradationPreference::MAINTAIN_RESOLUTION: {
asapersson13874762017-06-07 00:01:02 -07001644 // Scale up framerate.
1645 int fps = adaptation_request.framerate_fps_;
1646 if (adapt_counter.FramerateCount() == 1) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001647 RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting.";
asapersson13874762017-06-07 00:01:02 -07001648 fps = std::numeric_limits<int>::max();
sprangc5d62e22017-04-02 23:53:04 -07001649 }
sprangfda496a2017-06-15 04:21:07 -07001650
1651 const int requested_framerate =
1652 source_proxy_->RequestHigherFramerateThan(fps);
1653 if (requested_framerate == -1) {
Niels Möller7dc26b72017-12-06 10:27:48 +01001654 overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
asapersson13874762017-06-07 00:01:02 -07001655 return;
sprangfda496a2017-06-15 04:21:07 -07001656 }
Niels Möller7dc26b72017-12-06 10:27:48 +01001657 overuse_detector_->OnTargetFramerateUpdated(
1658 std::min(max_framerate_, requested_framerate));
asaperssonf7e294d2017-06-13 23:25:22 -07001659 GetAdaptCounter().DecrementFramerate(reason);
sprangc5d62e22017-04-02 23:53:04 -07001660 break;
asapersson13874762017-06-07 00:01:02 -07001661 }
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001662 case DegradationPreference::DISABLED:
asaperssonf7e294d2017-06-13 23:25:22 -07001663 return;
sprangc5d62e22017-04-02 23:53:04 -07001664 }
1665
asaperssond0de2952017-04-21 01:47:31 -07001666 last_adaptation_request_.emplace(adaptation_request);
1667
asapersson09f05612017-05-15 23:40:18 -07001668 UpdateAdaptationStats(reason);
1669
Mirko Bonadei675513b2017-11-09 11:09:25 +01001670 RTC_LOG(LS_INFO) << adapt_counter.ToString();
asapersson09f05612017-05-15 23:40:18 -07001671}
1672
Niels Möller213618e2018-07-24 09:29:58 +02001673// TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged.
mflodmancc3d4422017-08-03 08:27:51 -07001674void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) {
asaperssond0de2952017-04-21 01:47:31 -07001675 switch (reason) {
asaperssond0de2952017-04-21 01:47:31 -07001676 case kCpu:
Niels Möller213618e2018-07-24 09:29:58 +02001677 encoder_stats_observer_->OnAdaptationChanged(
1678 VideoStreamEncoderObserver::AdaptationReason::kCpu,
1679 GetActiveCounts(kCpu), GetActiveCounts(kQuality));
asapersson09f05612017-05-15 23:40:18 -07001680 break;
1681 case kQuality:
Niels Möller213618e2018-07-24 09:29:58 +02001682 encoder_stats_observer_->OnAdaptationChanged(
1683 VideoStreamEncoderObserver::AdaptationReason::kQuality,
1684 GetActiveCounts(kCpu), GetActiveCounts(kQuality));
asaperssond0de2952017-04-21 01:47:31 -07001685 break;
1686 }
perkj26091b12016-09-01 01:17:40 -07001687}
1688
Niels Möller213618e2018-07-24 09:29:58 +02001689VideoStreamEncoderObserver::AdaptationSteps VideoStreamEncoder::GetActiveCounts(
mflodmancc3d4422017-08-03 08:27:51 -07001690 AdaptReason reason) {
Niels Möller213618e2018-07-24 09:29:58 +02001691 VideoStreamEncoderObserver::AdaptationSteps counts =
mflodmancc3d4422017-08-03 08:27:51 -07001692 GetConstAdaptCounter().Counts(reason);
asapersson09f05612017-05-15 23:40:18 -07001693 switch (reason) {
1694 case kCpu:
1695 if (!IsFramerateScalingEnabled(degradation_preference_))
Niels Möller213618e2018-07-24 09:29:58 +02001696 counts.num_framerate_reductions = absl::nullopt;
asapersson09f05612017-05-15 23:40:18 -07001697 if (!IsResolutionScalingEnabled(degradation_preference_))
Niels Möller213618e2018-07-24 09:29:58 +02001698 counts.num_resolution_reductions = absl::nullopt;
asapersson09f05612017-05-15 23:40:18 -07001699 break;
1700 case kQuality:
1701 if (!IsFramerateScalingEnabled(degradation_preference_) ||
1702 !quality_scaler_) {
Niels Möller213618e2018-07-24 09:29:58 +02001703 counts.num_framerate_reductions = absl::nullopt;
asapersson09f05612017-05-15 23:40:18 -07001704 }
1705 if (!IsResolutionScalingEnabled(degradation_preference_) ||
1706 !quality_scaler_) {
Niels Möller213618e2018-07-24 09:29:58 +02001707 counts.num_resolution_reductions = absl::nullopt;
asapersson09f05612017-05-15 23:40:18 -07001708 }
1709 break;
sprangc5d62e22017-04-02 23:53:04 -07001710 }
asapersson09f05612017-05-15 23:40:18 -07001711 return counts;
sprangc5d62e22017-04-02 23:53:04 -07001712}
1713
mflodmancc3d4422017-08-03 08:27:51 -07001714VideoStreamEncoder::AdaptCounter& VideoStreamEncoder::GetAdaptCounter() {
asapersson09f05612017-05-15 23:40:18 -07001715 return adapt_counters_[degradation_preference_];
1716}
1717
mflodmancc3d4422017-08-03 08:27:51 -07001718const VideoStreamEncoder::AdaptCounter&
1719VideoStreamEncoder::GetConstAdaptCounter() {
asapersson09f05612017-05-15 23:40:18 -07001720 return adapt_counters_[degradation_preference_];
1721}
1722
Erik Språng7ca375c2019-02-06 16:20:17 +01001723void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image,
Niels Möller6bb5ab92019-01-11 11:11:10 +01001724 int64_t time_sent_us,
Erik Språng7ca375c2019-02-06 16:20:17 +01001725 int temporal_index) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001726 if (!encoder_queue_.IsCurrent()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01001727 encoder_queue_.PostTask(
1728 [this, encoded_image, time_sent_us, temporal_index] {
1729 RunPostEncode(encoded_image, time_sent_us, temporal_index);
1730 });
Niels Möller6bb5ab92019-01-11 11:11:10 +01001731 return;
1732 }
1733
1734 RTC_DCHECK_RUN_ON(&encoder_queue_);
Erik Språng7ca375c2019-02-06 16:20:17 +01001735
1736 absl::optional<int> encode_duration_us;
1737 if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) {
1738 encode_duration_us =
1739 // TODO(nisse): Maybe use capture_time_ms_ rather than encode_start_ms_?
1740 rtc::kNumMicrosecsPerMillisec *
1741 (encoded_image.timing_.encode_finish_ms -
1742 encoded_image.timing_.encode_start_ms);
1743 }
1744
1745 // Run post encode tasks, such as overuse detection and frame rate/drop
1746 // stats for internal encoders.
1747 const size_t frame_size = encoded_image.size();
Niels Möller87e2d782019-03-07 10:18:23 +01001748 const bool keyframe =
1749 encoded_image._frameType == VideoFrameType::kVideoFrameKey;
Erik Språng7ca375c2019-02-06 16:20:17 +01001750
1751 if (frame_size > 0) {
1752 frame_dropper_.Fill(frame_size, !keyframe);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001753 }
1754
Erik Språngd7329ca2019-02-21 21:19:53 +01001755 if (HasInternalSource()) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001756 // Update frame dropper after the fact for internal sources.
1757 input_framerate_.Update(1u, clock_->TimeInMilliseconds());
1758 frame_dropper_.Leak(GetInputFramerateFps());
1759 // Signal to encoder to drop next frame.
1760 if (frame_dropper_.DropFrame()) {
1761 pending_frame_drops_.fetch_add(1);
1762 }
1763 }
1764
Erik Språng7ca375c2019-02-06 16:20:17 +01001765 overuse_detector_->FrameSent(
1766 encoded_image.Timestamp(), time_sent_us,
1767 encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec,
1768 encode_duration_us);
1769 if (quality_scaler_ && encoded_image.qp_ >= 0)
Sebastian Janssonb6789402019-03-01 15:40:49 +01001770 quality_scaler_->ReportQp(encoded_image.qp_, time_sent_us);
Erik Språng7ca375c2019-02-06 16:20:17 +01001771 if (bitrate_adjuster_) {
1772 bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index);
1773 }
Niels Möller6bb5ab92019-01-11 11:11:10 +01001774}
1775
Erik Språngd7329ca2019-02-21 21:19:53 +01001776bool VideoStreamEncoder::HasInternalSource() const {
1777 // TODO(sprang): Checking both info from encoder and from encoder factory
1778 // until we have deprecated and removed the encoder factory info.
1779 return codec_info_.has_internal_source || encoder_info_.has_internal_source;
1780}
1781
Erik Språng6a7baa72019-02-26 18:31:00 +01001782void VideoStreamEncoder::ReleaseEncoder() {
1783 if (!encoder_ || !encoder_initialized_) {
1784 return;
1785 }
1786 encoder_->Release();
1787 encoder_initialized_ = false;
1788 TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release");
1789}
1790
asapersson09f05612017-05-15 23:40:18 -07001791// Class holding adaptation information.
mflodmancc3d4422017-08-03 08:27:51 -07001792VideoStreamEncoder::AdaptCounter::AdaptCounter() {
asapersson09f05612017-05-15 23:40:18 -07001793 fps_counters_.resize(kScaleReasonSize);
1794 resolution_counters_.resize(kScaleReasonSize);
asaperssonf7e294d2017-06-13 23:25:22 -07001795 static_assert(kScaleReasonSize == 2, "Update MoveCount.");
asapersson09f05612017-05-15 23:40:18 -07001796}
1797
mflodmancc3d4422017-08-03 08:27:51 -07001798VideoStreamEncoder::AdaptCounter::~AdaptCounter() {}
asapersson09f05612017-05-15 23:40:18 -07001799
mflodmancc3d4422017-08-03 08:27:51 -07001800std::string VideoStreamEncoder::AdaptCounter::ToString() const {
Jonas Olsson366a50c2018-09-06 13:41:30 +02001801 rtc::StringBuilder ss;
asapersson09f05612017-05-15 23:40:18 -07001802 ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
1803 ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
Jonas Olsson84df1c72018-09-14 16:59:32 +02001804 return ss.Release();
asapersson09f05612017-05-15 23:40:18 -07001805}
1806
Niels Möller213618e2018-07-24 09:29:58 +02001807VideoStreamEncoderObserver::AdaptationSteps
1808VideoStreamEncoder::AdaptCounter::Counts(int reason) const {
1809 VideoStreamEncoderObserver::AdaptationSteps counts;
1810 counts.num_framerate_reductions = fps_counters_[reason];
1811 counts.num_resolution_reductions = resolution_counters_[reason];
asapersson09f05612017-05-15 23:40:18 -07001812 return counts;
1813}
1814
mflodmancc3d4422017-08-03 08:27:51 -07001815void VideoStreamEncoder::AdaptCounter::IncrementFramerate(int reason) {
asaperssonf7e294d2017-06-13 23:25:22 -07001816 ++(fps_counters_[reason]);
asapersson09f05612017-05-15 23:40:18 -07001817}
1818
mflodmancc3d4422017-08-03 08:27:51 -07001819void VideoStreamEncoder::AdaptCounter::IncrementResolution(int reason) {
asaperssonf7e294d2017-06-13 23:25:22 -07001820 ++(resolution_counters_[reason]);
1821}
1822
mflodmancc3d4422017-08-03 08:27:51 -07001823void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason) {
asaperssonf7e294d2017-06-13 23:25:22 -07001824 if (fps_counters_[reason] == 0) {
1825 // Balanced mode: Adapt up is in a different order, switch reason.
1826 // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
1827 // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0}
1828 // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0}
1829 // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0}
1830 // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0}
1831 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
1832 RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
1833 MoveCount(&resolution_counters_, reason);
1834 MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
1835 }
1836 --(fps_counters_[reason]);
1837 RTC_DCHECK_GE(fps_counters_[reason], 0);
1838}
1839
mflodmancc3d4422017-08-03 08:27:51 -07001840void VideoStreamEncoder::AdaptCounter::DecrementResolution(int reason) {
asaperssonf7e294d2017-06-13 23:25:22 -07001841 if (resolution_counters_[reason] == 0) {
1842 // Balanced mode: Adapt up is in a different order, switch reason.
1843 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
1844 RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
1845 MoveCount(&fps_counters_, reason);
1846 MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
1847 }
1848 --(resolution_counters_[reason]);
1849 RTC_DCHECK_GE(resolution_counters_[reason], 0);
1850}
1851
mflodmancc3d4422017-08-03 08:27:51 -07001852void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason,
1853 int cur_fps) {
asaperssonf7e294d2017-06-13 23:25:22 -07001854 DecrementFramerate(reason);
1855 // Reset if at max fps (i.e. in case of fewer steps up than down).
1856 if (cur_fps == std::numeric_limits<int>::max())
1857 std::fill(fps_counters_.begin(), fps_counters_.end(), 0);
asapersson09f05612017-05-15 23:40:18 -07001858}
1859
mflodmancc3d4422017-08-03 08:27:51 -07001860int VideoStreamEncoder::AdaptCounter::FramerateCount() const {
asapersson09f05612017-05-15 23:40:18 -07001861 return Count(fps_counters_);
1862}
1863
mflodmancc3d4422017-08-03 08:27:51 -07001864int VideoStreamEncoder::AdaptCounter::ResolutionCount() const {
asapersson09f05612017-05-15 23:40:18 -07001865 return Count(resolution_counters_);
1866}
1867
mflodmancc3d4422017-08-03 08:27:51 -07001868int VideoStreamEncoder::AdaptCounter::FramerateCount(int reason) const {
asapersson09f05612017-05-15 23:40:18 -07001869 return fps_counters_[reason];
1870}
1871
mflodmancc3d4422017-08-03 08:27:51 -07001872int VideoStreamEncoder::AdaptCounter::ResolutionCount(int reason) const {
asapersson09f05612017-05-15 23:40:18 -07001873 return resolution_counters_[reason];
1874}
1875
mflodmancc3d4422017-08-03 08:27:51 -07001876int VideoStreamEncoder::AdaptCounter::TotalCount(int reason) const {
asapersson09f05612017-05-15 23:40:18 -07001877 return FramerateCount(reason) + ResolutionCount(reason);
1878}
1879
mflodmancc3d4422017-08-03 08:27:51 -07001880int VideoStreamEncoder::AdaptCounter::Count(
1881 const std::vector<int>& counters) const {
asapersson09f05612017-05-15 23:40:18 -07001882 return std::accumulate(counters.begin(), counters.end(), 0);
1883}
1884
mflodmancc3d4422017-08-03 08:27:51 -07001885void VideoStreamEncoder::AdaptCounter::MoveCount(std::vector<int>* counters,
1886 int from_reason) {
asaperssonf7e294d2017-06-13 23:25:22 -07001887 int to_reason = (from_reason + 1) % kScaleReasonSize;
1888 ++((*counters)[to_reason]);
1889 --((*counters)[from_reason]);
1890}
1891
mflodmancc3d4422017-08-03 08:27:51 -07001892std::string VideoStreamEncoder::AdaptCounter::ToString(
asapersson09f05612017-05-15 23:40:18 -07001893 const std::vector<int>& counters) const {
Jonas Olsson366a50c2018-09-06 13:41:30 +02001894 rtc::StringBuilder ss;
asapersson09f05612017-05-15 23:40:18 -07001895 for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
1896 ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
sprangc5d62e22017-04-02 23:53:04 -07001897 }
Jonas Olsson84df1c72018-09-14 16:59:32 +02001898 return ss.Release();
sprangc5d62e22017-04-02 23:53:04 -07001899}
1900
mflodman@webrtc.org84d17832011-12-01 17:02:23 +00001901} // namespace webrtc