blob: 4522300c9c5f8195a42a6528876c1dd4c3f1c5fd [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
stefan@webrtc.org07b45a52012-02-02 08:37:48 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Peter Boström7623ce42015-12-09 12:13:30 +010011#include "webrtc/video/vie_encoder.h"
mflodman@webrtc.org84d17832011-12-01 17:02:23 +000012
stefan@webrtc.orgc3cc3752013-06-04 09:36:56 +000013#include <algorithm>
perkj57c21f92016-06-17 07:27:16 -070014#include <limits>
sprangc5d62e22017-04-02 23:53:04 -070015#include <numeric>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
niklase@google.com470e71d2011-07-07 08:21:25 +000017
ilnik6b826ef2017-06-16 06:53:48 -070018#include "webrtc/api/video/i420_buffer.h"
kthelgason0cd27ba2016-12-19 06:32:16 -080019#include "webrtc/base/arraysize.h"
mflodman@webrtc.org02270cd2015-02-06 13:10:19 +000020#include "webrtc/base/checks.h"
tommidea489f2017-03-03 03:20:24 -080021#include "webrtc/base/location.h"
Peter Boström415d2cd2015-10-26 11:35:17 +010022#include "webrtc/base/logging.h"
Niels Möllerd28db7f2016-05-10 16:31:47 +020023#include "webrtc/base/timeutils.h"
tommidea489f2017-03-03 03:20:24 -080024#include "webrtc/base/trace_event.h"
sprang1a646ee2016-12-01 06:34:11 -080025#include "webrtc/common_video/include/video_bitrate_allocator.h"
nisseea3a7982017-05-15 02:42:11 -070026#include "webrtc/common_video/include/video_frame.h"
Henrik Kjellander0b9e29c2015-11-16 11:12:24 +010027#include "webrtc/modules/pacing/paced_sender.h"
Erik Språng08127a92016-11-16 16:41:30 +010028#include "webrtc/modules/video_coding/codecs/vp8/temporal_layers.h"
sprangb1ca0732017-02-01 08:38:12 -080029#include "webrtc/modules/video_coding/include/video_codec_initializer.h"
Henrik Kjellander2557b862015-11-18 22:00:21 +010030#include "webrtc/modules/video_coding/include/video_coding.h"
31#include "webrtc/modules/video_coding/include/video_coding_defines.h"
Peter Boströme4499152016-02-05 11:13:28 +010032#include "webrtc/video/overuse_frame_detector.h"
pbos@webrtc.org273a4142014-12-01 15:23:21 +000033#include "webrtc/video/send_statistics_proxy.h"
nisseea3a7982017-05-15 02:42:11 -070034
niklase@google.com470e71d2011-07-07 08:21:25 +000035namespace webrtc {
36
perkj26091b12016-09-01 01:17:40 -070037namespace {
sprangb1ca0732017-02-01 08:38:12 -080038
asapersson6ffb67d2016-09-12 00:10:45 -070039// Time interval for logging frame counts.
40const int64_t kFrameLogIntervalMs = 60000;
kthelgasonfa5fdce2017-02-27 00:15:31 -080041
kthelgason5e13d412016-12-01 03:59:51 -080042// We will never ask for a resolution lower than this.
kthelgason33ce8892016-12-09 03:53:59 -080043// TODO(kthelgason): Lower this limit when better testing
44// on MediaCodec and fallback implementations are in place.
kthelgasonfa5fdce2017-02-27 00:15:31 -080045// See https://bugs.chromium.org/p/webrtc/issues/detail?id=7206
kthelgason33ce8892016-12-09 03:53:59 -080046const int kMinPixelsPerFrame = 320 * 180;
sprangc5d62e22017-04-02 23:53:04 -070047const int kMinFramerateFps = 2;
sprangfda496a2017-06-15 04:21:07 -070048const int kMaxFramerateFps = 120;
perkj26091b12016-09-01 01:17:40 -070049
kthelgason2bc68642017-02-07 07:02:22 -080050// The maximum number of frames to drop at beginning of stream
51// to try and achieve desired bitrate.
52const int kMaxInitialFramedrop = 4;
53
kthelgason2bc68642017-02-07 07:02:22 -080054uint32_t MaximumFrameSizeForBitrate(uint32_t kbps) {
55 if (kbps > 0) {
56 if (kbps < 300 /* qvga */) {
57 return 320 * 240;
58 } else if (kbps < 500 /* vga */) {
59 return 640 * 480;
60 }
61 }
62 return std::numeric_limits<uint32_t>::max();
63}
64
asaperssonf7e294d2017-06-13 23:25:22 -070065// Initial limits for kBalanced degradation preference.
66int MinFps(int pixels) {
67 if (pixels <= 320 * 240) {
68 return 7;
69 } else if (pixels <= 480 * 270) {
70 return 10;
71 } else if (pixels <= 640 * 480) {
72 return 15;
73 } else {
74 return std::numeric_limits<int>::max();
75 }
76}
77
78int MaxFps(int pixels) {
79 if (pixels <= 320 * 240) {
80 return 10;
81 } else if (pixels <= 480 * 270) {
82 return 15;
83 } else {
84 return std::numeric_limits<int>::max();
85 }
86}
87
asapersson09f05612017-05-15 23:40:18 -070088bool IsResolutionScalingEnabled(
89 VideoSendStream::DegradationPreference degradation_preference) {
90 return degradation_preference ==
91 VideoSendStream::DegradationPreference::kMaintainFramerate ||
92 degradation_preference ==
93 VideoSendStream::DegradationPreference::kBalanced;
94}
95
96bool IsFramerateScalingEnabled(
97 VideoSendStream::DegradationPreference degradation_preference) {
98 return degradation_preference ==
99 VideoSendStream::DegradationPreference::kMaintainResolution ||
100 degradation_preference ==
101 VideoSendStream::DegradationPreference::kBalanced;
102}
103
perkj26091b12016-09-01 01:17:40 -0700104} // namespace
105
Pera48ddb72016-09-29 11:48:50 +0200106class ViEEncoder::ConfigureEncoderTask : public rtc::QueuedTask {
107 public:
108 ConfigureEncoderTask(ViEEncoder* vie_encoder,
109 VideoEncoderConfig config,
asapersson5f7226f2016-11-25 04:37:00 -0800110 size_t max_data_payload_length,
111 bool nack_enabled)
Pera48ddb72016-09-29 11:48:50 +0200112 : vie_encoder_(vie_encoder),
113 config_(std::move(config)),
asapersson5f7226f2016-11-25 04:37:00 -0800114 max_data_payload_length_(max_data_payload_length),
115 nack_enabled_(nack_enabled) {}
Pera48ddb72016-09-29 11:48:50 +0200116
117 private:
118 bool Run() override {
asapersson5f7226f2016-11-25 04:37:00 -0800119 vie_encoder_->ConfigureEncoderOnTaskQueue(
120 std::move(config_), max_data_payload_length_, nack_enabled_);
Pera48ddb72016-09-29 11:48:50 +0200121 return true;
122 }
123
124 ViEEncoder* const vie_encoder_;
125 VideoEncoderConfig config_;
126 size_t max_data_payload_length_;
asapersson5f7226f2016-11-25 04:37:00 -0800127 bool nack_enabled_;
Pera48ddb72016-09-29 11:48:50 +0200128};
129
perkj26091b12016-09-01 01:17:40 -0700130class ViEEncoder::EncodeTask : public rtc::QueuedTask {
131 public:
perkjd52063f2016-09-07 06:32:18 -0700132 EncodeTask(const VideoFrame& frame,
133 ViEEncoder* vie_encoder,
nissee0e3bdf2017-01-18 02:16:20 -0800134 int64_t time_when_posted_us,
asapersson6ffb67d2016-09-12 00:10:45 -0700135 bool log_stats)
nissedf2ceb82016-12-15 06:29:53 -0800136 : frame_(frame),
137 vie_encoder_(vie_encoder),
nissee0e3bdf2017-01-18 02:16:20 -0800138 time_when_posted_us_(time_when_posted_us),
asapersson6ffb67d2016-09-12 00:10:45 -0700139 log_stats_(log_stats) {
perkj26091b12016-09-01 01:17:40 -0700140 ++vie_encoder_->posted_frames_waiting_for_encode_;
141 }
142
143 private:
144 bool Run() override {
asapersson6ffb67d2016-09-12 00:10:45 -0700145 RTC_DCHECK_RUN_ON(&vie_encoder_->encoder_queue_);
perkj26091b12016-09-01 01:17:40 -0700146 RTC_DCHECK_GT(vie_encoder_->posted_frames_waiting_for_encode_.Value(), 0);
perkj803d97f2016-11-01 11:45:46 -0700147 vie_encoder_->stats_proxy_->OnIncomingFrame(frame_.width(),
148 frame_.height());
asapersson6ffb67d2016-09-12 00:10:45 -0700149 ++vie_encoder_->captured_frame_count_;
perkj26091b12016-09-01 01:17:40 -0700150 if (--vie_encoder_->posted_frames_waiting_for_encode_ == 0) {
nissee0e3bdf2017-01-18 02:16:20 -0800151 vie_encoder_->EncodeVideoFrame(frame_, time_when_posted_us_);
perkj26091b12016-09-01 01:17:40 -0700152 } else {
153 // There is a newer frame in flight. Do not encode this frame.
154 LOG(LS_VERBOSE)
155 << "Incoming frame dropped due to that the encoder is blocked.";
asapersson6ffb67d2016-09-12 00:10:45 -0700156 ++vie_encoder_->dropped_frame_count_;
157 }
158 if (log_stats_) {
159 LOG(LS_INFO) << "Number of frames: captured "
160 << vie_encoder_->captured_frame_count_
161 << ", dropped (due to encoder blocked) "
162 << vie_encoder_->dropped_frame_count_ << ", interval_ms "
163 << kFrameLogIntervalMs;
164 vie_encoder_->captured_frame_count_ = 0;
165 vie_encoder_->dropped_frame_count_ = 0;
perkj26091b12016-09-01 01:17:40 -0700166 }
167 return true;
168 }
169 VideoFrame frame_;
perkjd52063f2016-09-07 06:32:18 -0700170 ViEEncoder* const vie_encoder_;
nissee0e3bdf2017-01-18 02:16:20 -0800171 const int64_t time_when_posted_us_;
asapersson6ffb67d2016-09-12 00:10:45 -0700172 const bool log_stats_;
perkj26091b12016-09-01 01:17:40 -0700173};
174
perkja49cbd32016-09-16 07:53:41 -0700175// VideoSourceProxy is responsible ensuring thread safety between calls to
perkj803d97f2016-11-01 11:45:46 -0700176// ViEEncoder::SetSource that will happen on libjingle's worker thread when a
perkja49cbd32016-09-16 07:53:41 -0700177// video capturer is connected to the encoder and the encoder task queue
178// (encoder_queue_) where the encoder reports its VideoSinkWants.
179class ViEEncoder::VideoSourceProxy {
180 public:
181 explicit VideoSourceProxy(ViEEncoder* vie_encoder)
perkj803d97f2016-11-01 11:45:46 -0700182 : vie_encoder_(vie_encoder),
hbos8d609f62017-04-10 07:39:05 -0700183 degradation_preference_(
184 VideoSendStream::DegradationPreference::kDegradationDisabled),
perkj803d97f2016-11-01 11:45:46 -0700185 source_(nullptr) {}
perkja49cbd32016-09-16 07:53:41 -0700186
hbos8d609f62017-04-10 07:39:05 -0700187 void SetSource(
188 rtc::VideoSourceInterface<VideoFrame>* source,
189 const VideoSendStream::DegradationPreference& degradation_preference) {
perkj803d97f2016-11-01 11:45:46 -0700190 // Called on libjingle's worker thread.
perkja49cbd32016-09-16 07:53:41 -0700191 RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_);
192 rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr;
perkj803d97f2016-11-01 11:45:46 -0700193 rtc::VideoSinkWants wants;
perkja49cbd32016-09-16 07:53:41 -0700194 {
195 rtc::CritScope lock(&crit_);
sprangc5d62e22017-04-02 23:53:04 -0700196 degradation_preference_ = degradation_preference;
perkja49cbd32016-09-16 07:53:41 -0700197 old_source = source_;
198 source_ = source;
sprangfda496a2017-06-15 04:21:07 -0700199 wants = GetActiveSinkWantsInternal();
perkja49cbd32016-09-16 07:53:41 -0700200 }
201
202 if (old_source != source && old_source != nullptr) {
203 old_source->RemoveSink(vie_encoder_);
204 }
205
206 if (!source) {
207 return;
208 }
209
perkja49cbd32016-09-16 07:53:41 -0700210 source->AddOrUpdateSink(vie_encoder_, wants);
211 }
212
perkj803d97f2016-11-01 11:45:46 -0700213 void SetWantsRotationApplied(bool rotation_applied) {
214 rtc::CritScope lock(&crit_);
215 sink_wants_.rotation_applied = rotation_applied;
sprangc5d62e22017-04-02 23:53:04 -0700216 if (source_)
217 source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
218 }
219
sprangfda496a2017-06-15 04:21:07 -0700220 rtc::VideoSinkWants GetActiveSinkWants() {
221 rtc::CritScope lock(&crit_);
222 return GetActiveSinkWantsInternal();
perkj803d97f2016-11-01 11:45:46 -0700223 }
224
asaperssonf7e294d2017-06-13 23:25:22 -0700225 void ResetPixelFpsCount() {
226 rtc::CritScope lock(&crit_);
227 sink_wants_.max_pixel_count = std::numeric_limits<int>::max();
228 sink_wants_.target_pixel_count.reset();
229 sink_wants_.max_framerate_fps = std::numeric_limits<int>::max();
230 if (source_)
231 source_->AddOrUpdateSink(vie_encoder_, sink_wants_);
232 }
233
asaperssond0de2952017-04-21 01:47:31 -0700234 bool RequestResolutionLowerThan(int pixel_count) {
perkj803d97f2016-11-01 11:45:46 -0700235 // Called on the encoder task queue.
236 rtc::CritScope lock(&crit_);
asapersson13874762017-06-07 00:01:02 -0700237 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
asapersson02465b82017-04-10 01:12:52 -0700238 // This can happen since |degradation_preference_| is set on libjingle's
239 // worker thread but the adaptation is done on the encoder task queue.
asaperssond0de2952017-04-21 01:47:31 -0700240 return false;
perkj803d97f2016-11-01 11:45:46 -0700241 }
asapersson13874762017-06-07 00:01:02 -0700242 // The input video frame size will have a resolution less than or equal to
243 // |max_pixel_count| depending on how the source can scale the frame size.
kthelgason5e13d412016-12-01 03:59:51 -0800244 const int pixels_wanted = (pixel_count * 3) / 5;
asapersson13874762017-06-07 00:01:02 -0700245 if (pixels_wanted < kMinPixelsPerFrame ||
246 pixels_wanted >= sink_wants_.max_pixel_count) {
asaperssond0de2952017-04-21 01:47:31 -0700247 return false;
asapersson13874762017-06-07 00:01:02 -0700248 }
249 LOG(LS_INFO) << "Scaling down resolution, max pixels: " << pixels_wanted;
sprangc5d62e22017-04-02 23:53:04 -0700250 sink_wants_.max_pixel_count = pixels_wanted;
sprang84a37592017-02-10 07:04:27 -0800251 sink_wants_.target_pixel_count = rtc::Optional<int>();
sprangfda496a2017-06-15 04:21:07 -0700252 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal());
asaperssond0de2952017-04-21 01:47:31 -0700253 return true;
sprangc5d62e22017-04-02 23:53:04 -0700254 }
255
sprangfda496a2017-06-15 04:21:07 -0700256 int RequestFramerateLowerThan(int fps) {
sprangc5d62e22017-04-02 23:53:04 -0700257 // Called on the encoder task queue.
asapersson13874762017-06-07 00:01:02 -0700258 // The input video frame rate will be scaled down to 2/3, rounding down.
sprangfda496a2017-06-15 04:21:07 -0700259 int framerate_wanted = (fps * 2) / 3;
260 return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1;
perkj803d97f2016-11-01 11:45:46 -0700261 }
262
asapersson13874762017-06-07 00:01:02 -0700263 bool RequestHigherResolutionThan(int pixel_count) {
264 // Called on the encoder task queue.
perkj803d97f2016-11-01 11:45:46 -0700265 rtc::CritScope lock(&crit_);
asapersson13874762017-06-07 00:01:02 -0700266 if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) {
asapersson02465b82017-04-10 01:12:52 -0700267 // This can happen since |degradation_preference_| is set on libjingle's
268 // worker thread but the adaptation is done on the encoder task queue.
asapersson13874762017-06-07 00:01:02 -0700269 return false;
perkj803d97f2016-11-01 11:45:46 -0700270 }
asapersson13874762017-06-07 00:01:02 -0700271 int max_pixels_wanted = pixel_count;
272 if (max_pixels_wanted != std::numeric_limits<int>::max())
273 max_pixels_wanted = pixel_count * 4;
sprangc5d62e22017-04-02 23:53:04 -0700274
asapersson13874762017-06-07 00:01:02 -0700275 if (max_pixels_wanted <= sink_wants_.max_pixel_count)
276 return false;
277
278 sink_wants_.max_pixel_count = max_pixels_wanted;
279 if (max_pixels_wanted == std::numeric_limits<int>::max()) {
sprangc5d62e22017-04-02 23:53:04 -0700280 // Remove any constraints.
281 sink_wants_.target_pixel_count.reset();
sprangc5d62e22017-04-02 23:53:04 -0700282 } else {
283 // On step down we request at most 3/5 the pixel count of the previous
284 // resolution, so in order to take "one step up" we request a resolution
285 // as close as possible to 5/3 of the current resolution. The actual pixel
286 // count selected depends on the capabilities of the source. In order to
287 // not take a too large step up, we cap the requested pixel count to be at
288 // most four time the current number of pixels.
289 sink_wants_.target_pixel_count =
290 rtc::Optional<int>((pixel_count * 5) / 3);
sprangc5d62e22017-04-02 23:53:04 -0700291 }
asapersson13874762017-06-07 00:01:02 -0700292 LOG(LS_INFO) << "Scaling up resolution, max pixels: " << max_pixels_wanted;
sprangfda496a2017-06-15 04:21:07 -0700293 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal());
asapersson13874762017-06-07 00:01:02 -0700294 return true;
sprangc5d62e22017-04-02 23:53:04 -0700295 }
296
sprangfda496a2017-06-15 04:21:07 -0700297 // Request upgrade in framerate. Returns the new requested frame, or -1 if
298 // no change requested. Note that maxint may be returned if limits due to
299 // adaptation requests are removed completely. In that case, consider
300 // |max_framerate_| to be the current limit (assuming the capturer complies).
301 int RequestHigherFramerateThan(int fps) {
asapersson13874762017-06-07 00:01:02 -0700302 // Called on the encoder task queue.
303 // The input frame rate will be scaled up to the last step, with rounding.
304 int framerate_wanted = fps;
305 if (fps != std::numeric_limits<int>::max())
306 framerate_wanted = (fps * 3) / 2;
307
sprangfda496a2017-06-15 04:21:07 -0700308 return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1;
asapersson13874762017-06-07 00:01:02 -0700309 }
310
311 bool RestrictFramerate(int fps) {
sprangc5d62e22017-04-02 23:53:04 -0700312 // Called on the encoder task queue.
313 rtc::CritScope lock(&crit_);
asapersson13874762017-06-07 00:01:02 -0700314 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
315 return false;
316
317 const int fps_wanted = std::max(kMinFramerateFps, fps);
318 if (fps_wanted >= sink_wants_.max_framerate_fps)
319 return false;
320
321 LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted;
322 sink_wants_.max_framerate_fps = fps_wanted;
sprangfda496a2017-06-15 04:21:07 -0700323 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal());
asapersson13874762017-06-07 00:01:02 -0700324 return true;
325 }
326
327 bool IncreaseFramerate(int fps) {
328 // Called on the encoder task queue.
329 rtc::CritScope lock(&crit_);
330 if (!source_ || !IsFramerateScalingEnabled(degradation_preference_))
331 return false;
332
333 const int fps_wanted = std::max(kMinFramerateFps, fps);
334 if (fps_wanted <= sink_wants_.max_framerate_fps)
335 return false;
336
337 LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted;
338 sink_wants_.max_framerate_fps = fps_wanted;
sprangfda496a2017-06-15 04:21:07 -0700339 source_->AddOrUpdateSink(vie_encoder_, GetActiveSinkWantsInternal());
asapersson13874762017-06-07 00:01:02 -0700340 return true;
perkj803d97f2016-11-01 11:45:46 -0700341 }
342
perkja49cbd32016-09-16 07:53:41 -0700343 private:
sprangfda496a2017-06-15 04:21:07 -0700344 rtc::VideoSinkWants GetActiveSinkWantsInternal()
345 EXCLUSIVE_LOCKS_REQUIRED(&crit_) {
346 rtc::VideoSinkWants wants = sink_wants_;
347 // Clear any constraints from the current sink wants that don't apply to
348 // the used degradation_preference.
349 switch (degradation_preference_) {
350 case VideoSendStream::DegradationPreference::kBalanced:
351 break;
352 case VideoSendStream::DegradationPreference::kMaintainFramerate:
353 wants.max_framerate_fps = std::numeric_limits<int>::max();
354 break;
355 case VideoSendStream::DegradationPreference::kMaintainResolution:
356 wants.max_pixel_count = std::numeric_limits<int>::max();
357 wants.target_pixel_count.reset();
358 break;
359 case VideoSendStream::DegradationPreference::kDegradationDisabled:
360 wants.max_pixel_count = std::numeric_limits<int>::max();
361 wants.target_pixel_count.reset();
362 wants.max_framerate_fps = std::numeric_limits<int>::max();
363 }
364 return wants;
365 }
366
perkja49cbd32016-09-16 07:53:41 -0700367 rtc::CriticalSection crit_;
368 rtc::SequencedTaskChecker main_checker_;
perkj803d97f2016-11-01 11:45:46 -0700369 ViEEncoder* const vie_encoder_;
370 rtc::VideoSinkWants sink_wants_ GUARDED_BY(&crit_);
hbos8d609f62017-04-10 07:39:05 -0700371 VideoSendStream::DegradationPreference degradation_preference_
372 GUARDED_BY(&crit_);
perkja49cbd32016-09-16 07:53:41 -0700373 rtc::VideoSourceInterface<VideoFrame>* source_ GUARDED_BY(&crit_);
374
375 RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy);
376};
377
mflodman0dbf0092015-10-19 08:12:12 -0700378ViEEncoder::ViEEncoder(uint32_t number_of_cores,
Peter Boström7083e112015-09-22 16:28:51 +0200379 SendStatisticsProxy* stats_proxy,
perkj26091b12016-09-01 01:17:40 -0700380 const VideoSendStream::Config::EncoderSettings& settings,
381 rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
sprangfda496a2017-06-15 04:21:07 -0700382 EncodedFrameObserver* encoder_timing,
383 std::unique_ptr<OveruseFrameDetector> overuse_detector)
perkj26091b12016-09-01 01:17:40 -0700384 : shutdown_event_(true /* manual_reset */, false),
385 number_of_cores_(number_of_cores),
kthelgason2bc68642017-02-07 07:02:22 -0800386 initial_rampup_(0),
perkja49cbd32016-09-16 07:53:41 -0700387 source_proxy_(new VideoSourceProxy(this)),
Pera48ddb72016-09-29 11:48:50 +0200388 sink_(nullptr),
perkj26091b12016-09-01 01:17:40 -0700389 settings_(settings),
Erik Språng08127a92016-11-16 16:41:30 +0100390 codec_type_(PayloadNameToCodecType(settings.payload_name)
391 .value_or(VideoCodecType::kVideoCodecUnknown)),
perkjf5b2e512016-07-05 08:34:04 -0700392 video_sender_(Clock::GetRealTimeClock(), this, this),
sprangfda496a2017-06-15 04:21:07 -0700393 overuse_detector_(
394 overuse_detector.get()
395 ? overuse_detector.release()
396 : new OveruseFrameDetector(
397 GetCpuOveruseOptions(settings.full_overuse_time),
398 this,
399 encoder_timing,
400 stats_proxy)),
Peter Boström7083e112015-09-22 16:28:51 +0200401 stats_proxy_(stats_proxy),
perkj26091b12016-09-01 01:17:40 -0700402 pre_encode_callback_(pre_encode_callback),
403 module_process_thread_(nullptr),
sprangfda496a2017-06-15 04:21:07 -0700404 max_framerate_(-1),
perkjfa10b552016-10-02 23:45:26 -0700405 pending_encoder_reconfiguration_(false),
perkj26091b12016-09-01 01:17:40 -0700406 encoder_start_bitrate_bps_(0),
Pera48ddb72016-09-29 11:48:50 +0200407 max_data_payload_length_(0),
asapersson5f7226f2016-11-25 04:37:00 -0800408 nack_enabled_(false),
pbos@webrtc.org143451d2015-03-18 14:40:03 +0000409 last_observed_bitrate_bps_(0),
stefan@webrtc.org792f1a12015-03-04 12:24:26 +0000410 encoder_paused_and_dropped_frame_(false),
perkj26091b12016-09-01 01:17:40 -0700411 clock_(Clock::GetRealTimeClock()),
hbos8d609f62017-04-10 07:39:05 -0700412 degradation_preference_(
413 VideoSendStream::DegradationPreference::kDegradationDisabled),
perkj26091b12016-09-01 01:17:40 -0700414 last_captured_timestamp_(0),
415 delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() -
416 clock_->TimeInMilliseconds()),
asapersson6ffb67d2016-09-12 00:10:45 -0700417 last_frame_log_ms_(clock_->TimeInMilliseconds()),
418 captured_frame_count_(0),
419 dropped_frame_count_(0),
sprang1a646ee2016-12-01 06:34:11 -0800420 bitrate_observer_(nullptr),
perkj26091b12016-09-01 01:17:40 -0700421 encoder_queue_("EncoderQueue") {
sprang552c7c72017-02-13 04:41:45 -0800422 RTC_DCHECK(stats_proxy);
perkj803d97f2016-11-01 11:45:46 -0700423 encoder_queue_.PostTask([this] {
perkj26091b12016-09-01 01:17:40 -0700424 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangfda496a2017-06-15 04:21:07 -0700425 overuse_detector_->StartCheckForOveruse();
perkj26091b12016-09-01 01:17:40 -0700426 video_sender_.RegisterExternalEncoder(
427 settings_.encoder, settings_.payload_type, settings_.internal_source);
428 });
mflodman@webrtc.org02270cd2015-02-06 13:10:19 +0000429}
430
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000431ViEEncoder::~ViEEncoder() {
perkja49cbd32016-09-16 07:53:41 -0700432 RTC_DCHECK_RUN_ON(&thread_checker_);
perkj26091b12016-09-01 01:17:40 -0700433 RTC_DCHECK(shutdown_event_.Wait(0))
434 << "Must call ::Stop() before destruction.";
435}
436
sprangfda496a2017-06-15 04:21:07 -0700437// TODO(pbos): Lower these thresholds (to closer to 100%) when we handle
438// pipelining encoders better (multiple input frames before something comes
439// out). This should effectively turn off CPU adaptations for systems that
440// remotely cope with the load right now.
441CpuOveruseOptions ViEEncoder::GetCpuOveruseOptions(bool full_overuse_time) {
442 CpuOveruseOptions options;
443 if (full_overuse_time) {
444 options.low_encode_usage_threshold_percent = 150;
445 options.high_encode_usage_threshold_percent = 200;
446 }
447 return options;
448}
449
perkj26091b12016-09-01 01:17:40 -0700450void ViEEncoder::Stop() {
perkja49cbd32016-09-16 07:53:41 -0700451 RTC_DCHECK_RUN_ON(&thread_checker_);
hbos8d609f62017-04-10 07:39:05 -0700452 source_proxy_->SetSource(nullptr, VideoSendStream::DegradationPreference());
perkja49cbd32016-09-16 07:53:41 -0700453 encoder_queue_.PostTask([this] {
454 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangfda496a2017-06-15 04:21:07 -0700455 overuse_detector_->StopCheckForOveruse();
Erik Språng08127a92016-11-16 16:41:30 +0100456 rate_allocator_.reset();
sprang1a646ee2016-12-01 06:34:11 -0800457 bitrate_observer_ = nullptr;
perkja49cbd32016-09-16 07:53:41 -0700458 video_sender_.RegisterExternalEncoder(nullptr, settings_.payload_type,
459 false);
kthelgason876222f2016-11-29 01:44:11 -0800460 quality_scaler_ = nullptr;
perkja49cbd32016-09-16 07:53:41 -0700461 shutdown_event_.Set();
462 });
463
464 shutdown_event_.Wait(rtc::Event::kForever);
perkj26091b12016-09-01 01:17:40 -0700465}
466
467void ViEEncoder::RegisterProcessThread(ProcessThread* module_process_thread) {
perkja49cbd32016-09-16 07:53:41 -0700468 RTC_DCHECK_RUN_ON(&thread_checker_);
perkj26091b12016-09-01 01:17:40 -0700469 RTC_DCHECK(!module_process_thread_);
470 module_process_thread_ = module_process_thread;
tommidea489f2017-03-03 03:20:24 -0800471 module_process_thread_->RegisterModule(&video_sender_, RTC_FROM_HERE);
perkj26091b12016-09-01 01:17:40 -0700472 module_process_thread_checker_.DetachFromThread();
473}
474
475void ViEEncoder::DeRegisterProcessThread() {
perkja49cbd32016-09-16 07:53:41 -0700476 RTC_DCHECK_RUN_ON(&thread_checker_);
Peter Boströmcd5c25c2016-04-21 16:48:08 +0200477 module_process_thread_->DeRegisterModule(&video_sender_);
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000478}
479
sprang1a646ee2016-12-01 06:34:11 -0800480void ViEEncoder::SetBitrateObserver(
481 VideoBitrateAllocationObserver* bitrate_observer) {
482 RTC_DCHECK_RUN_ON(&thread_checker_);
483 encoder_queue_.PostTask([this, bitrate_observer] {
484 RTC_DCHECK_RUN_ON(&encoder_queue_);
485 RTC_DCHECK(!bitrate_observer_);
486 bitrate_observer_ = bitrate_observer;
487 });
488}
489
perkj803d97f2016-11-01 11:45:46 -0700490void ViEEncoder::SetSource(
491 rtc::VideoSourceInterface<VideoFrame>* source,
asapersson09f05612017-05-15 23:40:18 -0700492 const VideoSendStream::DegradationPreference& degradation_preference) {
perkja49cbd32016-09-16 07:53:41 -0700493 RTC_DCHECK_RUN_ON(&thread_checker_);
perkj803d97f2016-11-01 11:45:46 -0700494 source_proxy_->SetSource(source, degradation_preference);
495 encoder_queue_.PostTask([this, degradation_preference] {
496 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangc5d62e22017-04-02 23:53:04 -0700497 if (degradation_preference_ != degradation_preference) {
498 // Reset adaptation state, so that we're not tricked into thinking there's
499 // an already pending request of the same type.
500 last_adaptation_request_.reset();
asaperssonf7e294d2017-06-13 23:25:22 -0700501 if (degradation_preference ==
502 VideoSendStream::DegradationPreference::kBalanced ||
503 degradation_preference_ ==
504 VideoSendStream::DegradationPreference::kBalanced) {
505 // TODO(asapersson): Consider removing |adapt_counters_| map and use one
506 // AdaptCounter for all modes.
507 source_proxy_->ResetPixelFpsCount();
508 adapt_counters_.clear();
509 }
sprangc5d62e22017-04-02 23:53:04 -0700510 }
sprangb1ca0732017-02-01 08:38:12 -0800511 degradation_preference_ = degradation_preference;
asapersson91914e22017-06-01 00:34:08 -0700512 bool allow_scaling = IsResolutionScalingEnabled(degradation_preference_);
sprangc5d62e22017-04-02 23:53:04 -0700513 initial_rampup_ = allow_scaling ? 0 : kMaxInitialFramedrop;
kthelgason2bc68642017-02-07 07:02:22 -0800514 ConfigureQualityScaler();
sprangfda496a2017-06-15 04:21:07 -0700515 if (!IsFramerateScalingEnabled(degradation_preference) &&
516 max_framerate_ != -1) {
517 // If frame rate scaling is no longer allowed, remove any potential
518 // allowance for longer frame intervals.
519 overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
520 }
perkj803d97f2016-11-01 11:45:46 -0700521 });
perkja49cbd32016-09-16 07:53:41 -0700522}
523
perkj803d97f2016-11-01 11:45:46 -0700524void ViEEncoder::SetSink(EncoderSink* sink, bool rotation_applied) {
525 source_proxy_->SetWantsRotationApplied(rotation_applied);
perkj26091b12016-09-01 01:17:40 -0700526 encoder_queue_.PostTask([this, sink] {
527 RTC_DCHECK_RUN_ON(&encoder_queue_);
528 sink_ = sink;
529 });
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000530}
531
perkj26091b12016-09-01 01:17:40 -0700532void ViEEncoder::SetStartBitrate(int start_bitrate_bps) {
533 encoder_queue_.PostTask([this, start_bitrate_bps] {
534 RTC_DCHECK_RUN_ON(&encoder_queue_);
535 encoder_start_bitrate_bps_ = start_bitrate_bps;
536 });
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000537}
Peter Boström00b9d212016-05-19 16:59:03 +0200538
Per512ecb32016-09-23 15:52:06 +0200539void ViEEncoder::ConfigureEncoder(VideoEncoderConfig config,
asapersson5f7226f2016-11-25 04:37:00 -0800540 size_t max_data_payload_length,
541 bool nack_enabled) {
Pera48ddb72016-09-29 11:48:50 +0200542 encoder_queue_.PostTask(
543 std::unique_ptr<rtc::QueuedTask>(new ConfigureEncoderTask(
asapersson5f7226f2016-11-25 04:37:00 -0800544 this, std::move(config), max_data_payload_length, nack_enabled)));
perkj26091b12016-09-01 01:17:40 -0700545}
546
Pera48ddb72016-09-29 11:48:50 +0200547void ViEEncoder::ConfigureEncoderOnTaskQueue(VideoEncoderConfig config,
asapersson5f7226f2016-11-25 04:37:00 -0800548 size_t max_data_payload_length,
549 bool nack_enabled) {
perkj26091b12016-09-01 01:17:40 -0700550 RTC_DCHECK_RUN_ON(&encoder_queue_);
perkj26091b12016-09-01 01:17:40 -0700551 RTC_DCHECK(sink_);
perkjfa10b552016-10-02 23:45:26 -0700552 LOG(LS_INFO) << "ConfigureEncoder requested.";
Pera48ddb72016-09-29 11:48:50 +0200553
554 max_data_payload_length_ = max_data_payload_length;
asapersson5f7226f2016-11-25 04:37:00 -0800555 nack_enabled_ = nack_enabled;
Pera48ddb72016-09-29 11:48:50 +0200556 encoder_config_ = std::move(config);
perkjfa10b552016-10-02 23:45:26 -0700557 pending_encoder_reconfiguration_ = true;
Pera48ddb72016-09-29 11:48:50 +0200558
perkjfa10b552016-10-02 23:45:26 -0700559 // Reconfigure the encoder now if the encoder has an internal source or
Per21d45d22016-10-30 21:37:57 +0100560 // if the frame resolution is known. Otherwise, the reconfiguration is
561 // deferred until the next frame to minimize the number of reconfigurations.
562 // The codec configuration depends on incoming video frame size.
563 if (last_frame_info_) {
564 ReconfigureEncoder();
565 } else if (settings_.internal_source) {
brandtra3241662017-05-03 04:55:51 -0700566 last_frame_info_ =
567 rtc::Optional<VideoFrameInfo>(VideoFrameInfo(176, 144, false));
perkjfa10b552016-10-02 23:45:26 -0700568 ReconfigureEncoder();
569 }
570}
perkj26091b12016-09-01 01:17:40 -0700571
perkjfa10b552016-10-02 23:45:26 -0700572void ViEEncoder::ReconfigureEncoder() {
573 RTC_DCHECK_RUN_ON(&encoder_queue_);
574 RTC_DCHECK(pending_encoder_reconfiguration_);
575 std::vector<VideoStream> streams =
576 encoder_config_.video_stream_factory->CreateEncoderStreams(
577 last_frame_info_->width, last_frame_info_->height, encoder_config_);
perkj26091b12016-09-01 01:17:40 -0700578
ilnik6b826ef2017-06-16 06:53:48 -0700579 // TODO(ilnik): If configured resolution is significantly less than provided,
580 // e.g. because there are not enough SSRCs for all simulcast streams,
581 // signal new resolutions via SinkWants to video source.
582
583 // Stream dimensions may be not equal to given because of a simulcast
584 // restrictions.
585 int highest_stream_width = static_cast<int>(streams.back().width);
586 int highest_stream_height = static_cast<int>(streams.back().height);
587 // Dimension may be reduced to be, e.g. divisible by 4.
588 RTC_CHECK_GE(last_frame_info_->width, highest_stream_width);
589 RTC_CHECK_GE(last_frame_info_->height, highest_stream_height);
590 crop_width_ = last_frame_info_->width - highest_stream_width;
591 crop_height_ = last_frame_info_->height - highest_stream_height;
592
Erik Språng08127a92016-11-16 16:41:30 +0100593 VideoCodec codec;
594 if (!VideoCodecInitializer::SetupCodec(encoder_config_, settings_, streams,
asapersson5f7226f2016-11-25 04:37:00 -0800595 nack_enabled_, &codec,
596 &rate_allocator_)) {
Erik Språng08127a92016-11-16 16:41:30 +0100597 LOG(LS_ERROR) << "Failed to create encoder configuration.";
598 }
perkjfa10b552016-10-02 23:45:26 -0700599
600 codec.startBitrate =
601 std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate);
602 codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate);
603 codec.expect_encode_from_texture = last_frame_info_->is_texture;
sprangfda496a2017-06-15 04:21:07 -0700604 max_framerate_ = codec.maxFramerate;
605 RTC_DCHECK_LE(max_framerate_, kMaxFramerateFps);
Stefan Holmere5904162015-03-26 11:11:06 +0100606
Peter Boströmcd5c25c2016-04-21 16:48:08 +0200607 bool success = video_sender_.RegisterSendCodec(
perkjfa10b552016-10-02 23:45:26 -0700608 &codec, number_of_cores_,
609 static_cast<uint32_t>(max_data_payload_length_)) == VCM_OK;
Peter Boström905f8e72016-03-02 16:59:56 +0100610 if (!success) {
611 LOG(LS_ERROR) << "Failed to configure encoder.";
sprangfe627f32017-03-29 08:24:59 -0700612 rate_allocator_.reset();
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000613 }
Peter Boström905f8e72016-03-02 16:59:56 +0100614
ilnik35b7de42017-03-15 04:24:21 -0700615 video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
616 bitrate_observer_);
617
sprangfda496a2017-06-15 04:21:07 -0700618 // Get the current actual framerate, as measured by the stats proxy. This is
619 // used to get the correct bitrate layer allocation.
620 int current_framerate = stats_proxy_->GetSendFrameRate();
621 if (current_framerate == 0)
622 current_framerate = codec.maxFramerate;
sprang552c7c72017-02-13 04:41:45 -0800623 stats_proxy_->OnEncoderReconfigured(
sprangfda496a2017-06-15 04:21:07 -0700624 encoder_config_,
625 rate_allocator_.get()
626 ? rate_allocator_->GetPreferredBitrateBps(current_framerate)
627 : codec.maxBitrate);
Per512ecb32016-09-23 15:52:06 +0200628
perkjfa10b552016-10-02 23:45:26 -0700629 pending_encoder_reconfiguration_ = false;
Erik Språng08127a92016-11-16 16:41:30 +0100630
Pera48ddb72016-09-29 11:48:50 +0200631 sink_->OnEncoderConfigurationChanged(
perkjfa10b552016-10-02 23:45:26 -0700632 std::move(streams), encoder_config_.min_transmit_bitrate_bps);
kthelgason876222f2016-11-29 01:44:11 -0800633
sprangfda496a2017-06-15 04:21:07 -0700634 // Get the current target framerate, ie the maximum framerate as specified by
635 // the current codec configuration, or any limit imposed by cpu adaption in
636 // maintain-resolution or balanced mode. This is used to make sure overuse
637 // detection doesn't needlessly trigger in low and/or variable framerate
638 // scenarios.
639 int target_framerate = std::min(
640 max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps);
641 overuse_detector_->OnTargetFramerateUpdated(target_framerate);
642
kthelgason2bc68642017-02-07 07:02:22 -0800643 ConfigureQualityScaler();
644}
645
646void ViEEncoder::ConfigureQualityScaler() {
647 RTC_DCHECK_RUN_ON(&encoder_queue_);
kthelgason876222f2016-11-29 01:44:11 -0800648 const auto scaling_settings = settings_.encoder->GetScalingSettings();
asapersson36e9eb42017-03-31 05:29:12 -0700649 const bool quality_scaling_allowed =
asapersson91914e22017-06-01 00:34:08 -0700650 IsResolutionScalingEnabled(degradation_preference_) &&
651 scaling_settings.enabled;
kthelgason3af6cc02017-03-22 00:25:28 -0700652
asapersson36e9eb42017-03-31 05:29:12 -0700653 if (quality_scaling_allowed) {
asapersson09f05612017-05-15 23:40:18 -0700654 if (quality_scaler_.get() == nullptr) {
655 // Quality scaler has not already been configured.
656 // Drop frames and scale down until desired quality is achieved.
657 if (scaling_settings.thresholds) {
658 quality_scaler_.reset(
659 new QualityScaler(this, *(scaling_settings.thresholds)));
660 } else {
661 quality_scaler_.reset(new QualityScaler(this, codec_type_));
662 }
kthelgason876222f2016-11-29 01:44:11 -0800663 }
664 } else {
665 quality_scaler_.reset(nullptr);
kthelgasonad9010c2017-02-14 00:46:51 -0800666 initial_rampup_ = kMaxInitialFramedrop;
kthelgason876222f2016-11-29 01:44:11 -0800667 }
asapersson09f05612017-05-15 23:40:18 -0700668
669 stats_proxy_->SetAdaptationStats(GetActiveCounts(kCpu),
670 GetActiveCounts(kQuality));
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000671}
672
perkja49cbd32016-09-16 07:53:41 -0700673void ViEEncoder::OnFrame(const VideoFrame& video_frame) {
perkj26091b12016-09-01 01:17:40 -0700674 RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_);
perkj26091b12016-09-01 01:17:40 -0700675 VideoFrame incoming_frame = video_frame;
676
677 // Local time in webrtc time base.
ilnik04f4d122017-06-19 07:18:55 -0700678 int64_t current_time_us = clock_->TimeInMicroseconds();
679 int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec;
680 // In some cases, e.g., when the frame from decoder is fed to encoder,
681 // the timestamp may be set to the future. As the encoding pipeline assumes
682 // capture time to be less than present time, we should reset the capture
683 // timestamps here. Otherwise there may be issues with RTP send stream.
684 if (incoming_frame.timestamp_us() > current_time_us)
685 incoming_frame.set_timestamp_us(current_time_us);
perkj26091b12016-09-01 01:17:40 -0700686
687 // Capture time may come from clock with an offset and drift from clock_.
688 int64_t capture_ntp_time_ms;
nisse891419f2017-01-12 10:02:22 -0800689 if (video_frame.ntp_time_ms() > 0) {
perkj26091b12016-09-01 01:17:40 -0700690 capture_ntp_time_ms = video_frame.ntp_time_ms();
691 } else if (video_frame.render_time_ms() != 0) {
692 capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_;
693 } else {
nisse1c0dea82017-01-30 02:43:18 -0800694 capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_;
perkj26091b12016-09-01 01:17:40 -0700695 }
696 incoming_frame.set_ntp_time_ms(capture_ntp_time_ms);
697
698 // Convert NTP time, in ms, to RTP timestamp.
699 const int kMsToRtpTimestamp = 90;
700 incoming_frame.set_timestamp(
701 kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms()));
702
703 if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) {
704 // We don't allow the same capture time for two frames, drop this one.
705 LOG(LS_WARNING) << "Same/old NTP timestamp ("
706 << incoming_frame.ntp_time_ms()
707 << " <= " << last_captured_timestamp_
708 << ") for incoming frame. Dropping.";
709 return;
710 }
711
asapersson6ffb67d2016-09-12 00:10:45 -0700712 bool log_stats = false;
nisse1c0dea82017-01-30 02:43:18 -0800713 if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) {
714 last_frame_log_ms_ = current_time_ms;
asapersson6ffb67d2016-09-12 00:10:45 -0700715 log_stats = true;
716 }
717
perkj26091b12016-09-01 01:17:40 -0700718 last_captured_timestamp_ = incoming_frame.ntp_time_ms();
asapersson6ffb67d2016-09-12 00:10:45 -0700719 encoder_queue_.PostTask(std::unique_ptr<rtc::QueuedTask>(new EncodeTask(
nissee0e3bdf2017-01-18 02:16:20 -0800720 incoming_frame, this, rtc::TimeMicros(), log_stats)));
perkj26091b12016-09-01 01:17:40 -0700721}
722
stefan@webrtc.orgbfacda62013-03-27 16:36:01 +0000723bool ViEEncoder::EncoderPaused() const {
perkj26091b12016-09-01 01:17:40 -0700724 RTC_DCHECK_RUN_ON(&encoder_queue_);
pwestin@webrtc.org91563e42013-04-25 22:20:08 +0000725 // Pause video if paused by caller or as long as the network is down or the
726 // pacer queue has grown too large in buffered mode.
perkj57c21f92016-06-17 07:27:16 -0700727 // If the pacer queue has grown too large or the network is down,
perkjfea93092016-05-14 00:58:48 -0700728 // last_observed_bitrate_bps_ will be 0.
perkj26091b12016-09-01 01:17:40 -0700729 return last_observed_bitrate_bps_ == 0;
stefan@webrtc.orgbfacda62013-03-27 16:36:01 +0000730}
731
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000732void ViEEncoder::TraceFrameDropStart() {
perkj26091b12016-09-01 01:17:40 -0700733 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000734 // Start trace event only on the first frame after encoder is paused.
735 if (!encoder_paused_and_dropped_frame_) {
736 TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this);
737 }
738 encoder_paused_and_dropped_frame_ = true;
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000739}
740
741void ViEEncoder::TraceFrameDropEnd() {
perkj26091b12016-09-01 01:17:40 -0700742 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprang@webrtc.orgdcebf2d2014-11-04 16:27:16 +0000743 // End trace event on first frame after encoder resumes, if frame was dropped.
744 if (encoder_paused_and_dropped_frame_) {
745 TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this);
746 }
747 encoder_paused_and_dropped_frame_ = false;
748}
749
perkjd52063f2016-09-07 06:32:18 -0700750void ViEEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
nissee0e3bdf2017-01-18 02:16:20 -0800751 int64_t time_when_posted_us) {
perkj26091b12016-09-01 01:17:40 -0700752 RTC_DCHECK_RUN_ON(&encoder_queue_);
kthelgason876222f2016-11-29 01:44:11 -0800753
perkj26091b12016-09-01 01:17:40 -0700754 if (pre_encode_callback_)
755 pre_encode_callback_->OnFrame(video_frame);
756
Per21d45d22016-10-30 21:37:57 +0100757 if (!last_frame_info_ || video_frame.width() != last_frame_info_->width ||
perkjfa10b552016-10-02 23:45:26 -0700758 video_frame.height() != last_frame_info_->height ||
perkjfa10b552016-10-02 23:45:26 -0700759 video_frame.is_texture() != last_frame_info_->is_texture) {
760 pending_encoder_reconfiguration_ = true;
brandtra3241662017-05-03 04:55:51 -0700761 last_frame_info_ = rtc::Optional<VideoFrameInfo>(VideoFrameInfo(
762 video_frame.width(), video_frame.height(), video_frame.is_texture()));
perkjfa10b552016-10-02 23:45:26 -0700763 LOG(LS_INFO) << "Video frame parameters changed: dimensions="
764 << last_frame_info_->width << "x" << last_frame_info_->height
brandtra3241662017-05-03 04:55:51 -0700765 << ", texture=" << last_frame_info_->is_texture << ".";
perkjfa10b552016-10-02 23:45:26 -0700766 }
767
kthelgason2bc68642017-02-07 07:02:22 -0800768 if (initial_rampup_ < kMaxInitialFramedrop &&
769 video_frame.size() >
770 MaximumFrameSizeForBitrate(encoder_start_bitrate_bps_ / 1000)) {
771 LOG(LS_INFO) << "Dropping frame. Too large for target bitrate.";
772 AdaptDown(kQuality);
773 ++initial_rampup_;
774 return;
775 }
776 initial_rampup_ = kMaxInitialFramedrop;
777
sprang57c2fff2017-01-16 06:24:02 -0800778 int64_t now_ms = clock_->TimeInMilliseconds();
perkjfa10b552016-10-02 23:45:26 -0700779 if (pending_encoder_reconfiguration_) {
780 ReconfigureEncoder();
sprang4847ae62017-06-27 07:06:52 -0700781 last_parameters_update_ms_.emplace(now_ms);
sprang57c2fff2017-01-16 06:24:02 -0800782 } else if (!last_parameters_update_ms_ ||
783 now_ms - *last_parameters_update_ms_ >=
784 vcm::VCMProcessTimer::kDefaultProcessIntervalMs) {
785 video_sender_.UpdateChannelParemeters(rate_allocator_.get(),
786 bitrate_observer_);
sprang4847ae62017-06-27 07:06:52 -0700787 last_parameters_update_ms_.emplace(now_ms);
perkjfa10b552016-10-02 23:45:26 -0700788 }
789
perkj26091b12016-09-01 01:17:40 -0700790 if (EncoderPaused()) {
791 TraceFrameDropStart();
792 return;
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000793 }
perkj26091b12016-09-01 01:17:40 -0700794 TraceFrameDropEnd();
niklase@google.com470e71d2011-07-07 08:21:25 +0000795
ilnik6b826ef2017-06-16 06:53:48 -0700796 VideoFrame out_frame(video_frame);
797 // Crop frame if needed.
798 if (crop_width_ > 0 || crop_height_ > 0) {
799 int cropped_width = video_frame.width() - crop_width_;
800 int cropped_height = video_frame.height() - crop_height_;
801 rtc::scoped_refptr<I420Buffer> cropped_buffer =
802 I420Buffer::Create(cropped_width, cropped_height);
803 // TODO(ilnik): Remove scaling if cropping is too big, as it should never
804 // happen after SinkWants signaled correctly from ReconfigureEncoder.
805 if (crop_width_ < 4 && crop_height_ < 4) {
806 cropped_buffer->CropAndScaleFrom(
807 *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2,
808 crop_height_ / 2, cropped_width, cropped_height);
809 } else {
810 cropped_buffer->ScaleFrom(
811 *video_frame.video_frame_buffer()->ToI420().get());
812 }
813 out_frame =
814 VideoFrame(cropped_buffer, video_frame.timestamp(),
815 video_frame.render_time_ms(), video_frame.rotation());
816 out_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
817 }
818
Magnus Jedvert26679d62015-04-07 14:07:41 +0200819 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(),
hclam@chromium.org1a7b9b92013-07-08 21:31:18 +0000820 "Encode");
pbos@webrtc.orgfe1ef932013-10-21 10:34:43 +0000821
ilnik6b826ef2017-06-16 06:53:48 -0700822 overuse_detector_->FrameCaptured(out_frame, time_when_posted_us);
perkjd52063f2016-09-07 06:32:18 -0700823
ilnik6b826ef2017-06-16 06:53:48 -0700824 video_sender_.AddVideoFrame(out_frame, nullptr);
niklase@google.com470e71d2011-07-07 08:21:25 +0000825}
niklase@google.com470e71d2011-07-07 08:21:25 +0000826
Peter Boström233bfd22016-01-18 20:23:40 +0100827void ViEEncoder::SendKeyFrame() {
perkj26091b12016-09-01 01:17:40 -0700828 if (!encoder_queue_.IsCurrent()) {
829 encoder_queue_.PostTask([this] { SendKeyFrame(); });
830 return;
831 }
832 RTC_DCHECK_RUN_ON(&encoder_queue_);
Peter Boströmcd5c25c2016-04-21 16:48:08 +0200833 video_sender_.IntraFrameRequest(0);
stefan@webrtc.org07b45a52012-02-02 08:37:48 +0000834}
835
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700836EncodedImageCallback::Result ViEEncoder::OnEncodedImage(
837 const EncodedImage& encoded_image,
838 const CodecSpecificInfo* codec_specific_info,
839 const RTPFragmentationHeader* fragmentation) {
perkj26091b12016-09-01 01:17:40 -0700840 // Encoded is called on whatever thread the real encoder implementation run
841 // on. In the case of hardware encoders, there might be several encoders
842 // running in parallel on different threads.
sprang552c7c72017-02-13 04:41:45 -0800843 stats_proxy_->OnSendEncodedImage(encoded_image, codec_specific_info);
sprang3911c262016-04-15 01:24:14 -0700844
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700845 EncodedImageCallback::Result result =
846 sink_->OnEncodedImage(encoded_image, codec_specific_info, fragmentation);
perkjbc75d972016-05-02 06:31:25 -0700847
nissee0e3bdf2017-01-18 02:16:20 -0800848 int64_t time_sent_us = rtc::TimeMicros();
perkjd52063f2016-09-07 06:32:18 -0700849 uint32_t timestamp = encoded_image._timeStamp;
kthelgason876222f2016-11-29 01:44:11 -0800850 const int qp = encoded_image.qp_;
nissee0e3bdf2017-01-18 02:16:20 -0800851 encoder_queue_.PostTask([this, timestamp, time_sent_us, qp] {
perkjd52063f2016-09-07 06:32:18 -0700852 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangfda496a2017-06-15 04:21:07 -0700853 overuse_detector_->FrameSent(timestamp, time_sent_us);
sprang84a37592017-02-10 07:04:27 -0800854 if (quality_scaler_ && qp >= 0)
kthelgason876222f2016-11-29 01:44:11 -0800855 quality_scaler_->ReportQP(qp);
perkjd52063f2016-09-07 06:32:18 -0700856 });
perkj803d97f2016-11-01 11:45:46 -0700857
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700858 return result;
Peter Boströmb7d9a972015-12-18 16:01:11 +0100859}
860
kthelgason876222f2016-11-29 01:44:11 -0800861void ViEEncoder::OnDroppedFrame() {
862 encoder_queue_.PostTask([this] {
863 RTC_DCHECK_RUN_ON(&encoder_queue_);
864 if (quality_scaler_)
865 quality_scaler_->ReportDroppedFrame();
866 });
867}
868
perkj275afc52016-09-01 00:21:16 -0700869void ViEEncoder::SendStatistics(uint32_t bit_rate, uint32_t frame_rate) {
perkj26091b12016-09-01 01:17:40 -0700870 RTC_DCHECK(module_process_thread_checker_.CalledOnValidThread());
sprang552c7c72017-02-13 04:41:45 -0800871 stats_proxy_->OnEncoderStatsUpdate(frame_rate, bit_rate);
niklase@google.com470e71d2011-07-07 08:21:25 +0000872}
873
perkj600246e2016-05-04 11:26:51 -0700874void ViEEncoder::OnReceivedIntraFrameRequest(size_t stream_index) {
perkj26091b12016-09-01 01:17:40 -0700875 if (!encoder_queue_.IsCurrent()) {
876 encoder_queue_.PostTask(
877 [this, stream_index] { OnReceivedIntraFrameRequest(stream_index); });
878 return;
879 }
880 RTC_DCHECK_RUN_ON(&encoder_queue_);
mflodman@webrtc.org84d17832011-12-01 17:02:23 +0000881 // Key frame request from remote side, signal to VCM.
justinlin@chromium.org7bfb3a32013-05-13 22:59:00 +0000882 TRACE_EVENT0("webrtc", "OnKeyFrameRequest");
perkj600246e2016-05-04 11:26:51 -0700883 video_sender_.IntraFrameRequest(stream_index);
mflodman@webrtc.orgaca26292012-10-05 16:17:41 +0000884}
885
mflodman86aabb22016-03-11 15:44:32 +0100886void ViEEncoder::OnBitrateUpdated(uint32_t bitrate_bps,
stefan@webrtc.orgedeea912014-12-08 19:46:23 +0000887 uint8_t fraction_lost,
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000888 int64_t round_trip_time_ms) {
perkj26091b12016-09-01 01:17:40 -0700889 if (!encoder_queue_.IsCurrent()) {
890 encoder_queue_.PostTask(
891 [this, bitrate_bps, fraction_lost, round_trip_time_ms] {
892 OnBitrateUpdated(bitrate_bps, fraction_lost, round_trip_time_ms);
893 });
894 return;
895 }
896 RTC_DCHECK_RUN_ON(&encoder_queue_);
897 RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active.";
898
perkjec81bcd2016-05-11 06:01:13 -0700899 LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << bitrate_bps
mflodman8602a3d2015-05-20 15:54:42 -0700900 << " packet loss " << static_cast<int>(fraction_lost)
mflodman@webrtc.org5574dac2014-04-07 10:56:31 +0000901 << " rtt " << round_trip_time_ms;
perkj26091b12016-09-01 01:17:40 -0700902
Peter Boströmcd5c25c2016-04-21 16:48:08 +0200903 video_sender_.SetChannelParameters(bitrate_bps, fraction_lost,
sprang1a646ee2016-12-01 06:34:11 -0800904 round_trip_time_ms, rate_allocator_.get(),
905 bitrate_observer_);
perkj26091b12016-09-01 01:17:40 -0700906
907 encoder_start_bitrate_bps_ =
908 bitrate_bps != 0 ? bitrate_bps : encoder_start_bitrate_bps_;
mflodman101f2502016-06-09 17:21:19 +0200909 bool video_is_suspended = bitrate_bps == 0;
Erik Språng08127a92016-11-16 16:41:30 +0100910 bool video_suspension_changed = video_is_suspended != EncoderPaused();
perkj26091b12016-09-01 01:17:40 -0700911 last_observed_bitrate_bps_ = bitrate_bps;
Peter Boströmd153a372015-11-10 15:27:12 +0000912
sprang552c7c72017-02-13 04:41:45 -0800913 if (video_suspension_changed) {
mflodman522739c2016-06-22 17:42:30 +0200914 LOG(LS_INFO) << "Video suspend state changed to: "
915 << (video_is_suspended ? "suspended" : "not suspended");
Peter Boström7083e112015-09-22 16:28:51 +0200916 stats_proxy_->OnSuspendChange(video_is_suspended);
mflodman101f2502016-06-09 17:21:19 +0200917 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000918}
919
sprangb1ca0732017-02-01 08:38:12 -0800920void ViEEncoder::AdaptDown(AdaptReason reason) {
perkjd52063f2016-09-07 06:32:18 -0700921 RTC_DCHECK_RUN_ON(&encoder_queue_);
sprangc5d62e22017-04-02 23:53:04 -0700922 AdaptationRequest adaptation_request = {
923 last_frame_info_->pixel_count(),
924 stats_proxy_->GetStats().input_frame_rate,
925 AdaptationRequest::Mode::kAdaptDown};
asapersson09f05612017-05-15 23:40:18 -0700926
sprangc5d62e22017-04-02 23:53:04 -0700927 bool downgrade_requested =
928 last_adaptation_request_ &&
929 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown;
930
sprangc5d62e22017-04-02 23:53:04 -0700931 switch (degradation_preference_) {
hbos8d609f62017-04-10 07:39:05 -0700932 case VideoSendStream::DegradationPreference::kBalanced:
asaperssonf7e294d2017-06-13 23:25:22 -0700933 break;
hbos8d609f62017-04-10 07:39:05 -0700934 case VideoSendStream::DegradationPreference::kMaintainFramerate:
sprangc5d62e22017-04-02 23:53:04 -0700935 if (downgrade_requested &&
936 adaptation_request.input_pixel_count_ >=
937 last_adaptation_request_->input_pixel_count_) {
938 // Don't request lower resolution if the current resolution is not
939 // lower than the last time we asked for the resolution to be lowered.
940 return;
941 }
942 break;
hbos8d609f62017-04-10 07:39:05 -0700943 case VideoSendStream::DegradationPreference::kMaintainResolution:
sprangc5d62e22017-04-02 23:53:04 -0700944 if (adaptation_request.framerate_fps_ <= 0 ||
945 (downgrade_requested &&
946 adaptation_request.framerate_fps_ < kMinFramerateFps)) {
947 // If no input fps estimate available, can't determine how to scale down
948 // framerate. Otherwise, don't request lower framerate if we don't have
949 // a valid frame rate. Since framerate, unlike resolution, is a measure
950 // we have to estimate, and can fluctuate naturally over time, don't
951 // make the same kind of limitations as for resolution, but trust the
952 // overuse detector to not trigger too often.
953 return;
954 }
955 break;
hbos8d609f62017-04-10 07:39:05 -0700956 case VideoSendStream::DegradationPreference::kDegradationDisabled:
sprangc5d62e22017-04-02 23:53:04 -0700957 return;
sprang84a37592017-02-10 07:04:27 -0800958 }
sprangc5d62e22017-04-02 23:53:04 -0700959
asaperssond0de2952017-04-21 01:47:31 -0700960 if (reason == kCpu) {
asaperssonf7e294d2017-06-13 23:25:22 -0700961 if (GetConstAdaptCounter().ResolutionCount(kCpu) >=
962 kMaxCpuResolutionDowngrades ||
963 GetConstAdaptCounter().FramerateCount(kCpu) >=
964 kMaxCpuFramerateDowngrades) {
asaperssond0de2952017-04-21 01:47:31 -0700965 return;
asaperssonf7e294d2017-06-13 23:25:22 -0700966 }
kthelgason876222f2016-11-29 01:44:11 -0800967 }
sprangc5d62e22017-04-02 23:53:04 -0700968
sprangc5d62e22017-04-02 23:53:04 -0700969 switch (degradation_preference_) {
asaperssonf7e294d2017-06-13 23:25:22 -0700970 case VideoSendStream::DegradationPreference::kBalanced: {
971 // Try scale down framerate, if lower.
972 int fps = MinFps(last_frame_info_->pixel_count());
973 if (source_proxy_->RestrictFramerate(fps)) {
974 GetAdaptCounter().IncrementFramerate(reason);
975 break;
976 }
977 // Scale down resolution.
sprang317005a2017-06-08 07:12:17 -0700978 FALLTHROUGH();
asaperssonf7e294d2017-06-13 23:25:22 -0700979 }
hbos8d609f62017-04-10 07:39:05 -0700980 case VideoSendStream::DegradationPreference::kMaintainFramerate:
asapersson13874762017-06-07 00:01:02 -0700981 // Scale down resolution.
asaperssond0de2952017-04-21 01:47:31 -0700982 if (!source_proxy_->RequestResolutionLowerThan(
983 adaptation_request.input_pixel_count_)) {
984 return;
985 }
asaperssonf7e294d2017-06-13 23:25:22 -0700986 GetAdaptCounter().IncrementResolution(reason);
sprangc5d62e22017-04-02 23:53:04 -0700987 break;
sprangfda496a2017-06-15 04:21:07 -0700988 case VideoSendStream::DegradationPreference::kMaintainResolution: {
asapersson13874762017-06-07 00:01:02 -0700989 // Scale down framerate.
sprangfda496a2017-06-15 04:21:07 -0700990 const int requested_framerate = source_proxy_->RequestFramerateLowerThan(
991 adaptation_request.framerate_fps_);
992 if (requested_framerate == -1)
asapersson13874762017-06-07 00:01:02 -0700993 return;
sprangfda496a2017-06-15 04:21:07 -0700994 RTC_DCHECK_NE(max_framerate_, -1);
995 overuse_detector_->OnTargetFramerateUpdated(
996 std::min(max_framerate_, requested_framerate));
asaperssonf7e294d2017-06-13 23:25:22 -0700997 GetAdaptCounter().IncrementFramerate(reason);
sprangc5d62e22017-04-02 23:53:04 -0700998 break;
sprangfda496a2017-06-15 04:21:07 -0700999 }
hbos8d609f62017-04-10 07:39:05 -07001000 case VideoSendStream::DegradationPreference::kDegradationDisabled:
sprangc5d62e22017-04-02 23:53:04 -07001001 RTC_NOTREACHED();
1002 }
1003
asaperssond0de2952017-04-21 01:47:31 -07001004 last_adaptation_request_.emplace(adaptation_request);
1005
asapersson09f05612017-05-15 23:40:18 -07001006 UpdateAdaptationStats(reason);
asaperssond0de2952017-04-21 01:47:31 -07001007
asapersson09f05612017-05-15 23:40:18 -07001008 LOG(LS_INFO) << GetConstAdaptCounter().ToString();
perkj26091b12016-09-01 01:17:40 -07001009}
1010
sprangb1ca0732017-02-01 08:38:12 -08001011void ViEEncoder::AdaptUp(AdaptReason reason) {
perkjd52063f2016-09-07 06:32:18 -07001012 RTC_DCHECK_RUN_ON(&encoder_queue_);
asapersson09f05612017-05-15 23:40:18 -07001013
1014 const AdaptCounter& adapt_counter = GetConstAdaptCounter();
1015 int num_downgrades = adapt_counter.TotalCount(reason);
1016 if (num_downgrades == 0)
perkj803d97f2016-11-01 11:45:46 -07001017 return;
asapersson09f05612017-05-15 23:40:18 -07001018 RTC_DCHECK_GT(num_downgrades, 0);
1019
sprangc5d62e22017-04-02 23:53:04 -07001020 AdaptationRequest adaptation_request = {
1021 last_frame_info_->pixel_count(),
1022 stats_proxy_->GetStats().input_frame_rate,
1023 AdaptationRequest::Mode::kAdaptUp};
1024
1025 bool adapt_up_requested =
1026 last_adaptation_request_ &&
1027 last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp;
asapersson09f05612017-05-15 23:40:18 -07001028
asaperssonf7e294d2017-06-13 23:25:22 -07001029 if (degradation_preference_ ==
1030 VideoSendStream::DegradationPreference::kMaintainFramerate) {
1031 if (adapt_up_requested &&
1032 adaptation_request.input_pixel_count_ <=
1033 last_adaptation_request_->input_pixel_count_) {
1034 // Don't request higher resolution if the current resolution is not
1035 // higher than the last time we asked for the resolution to be higher.
sprangc5d62e22017-04-02 23:53:04 -07001036 return;
asaperssonf7e294d2017-06-13 23:25:22 -07001037 }
sprangb1ca0732017-02-01 08:38:12 -08001038 }
sprangc5d62e22017-04-02 23:53:04 -07001039
sprangc5d62e22017-04-02 23:53:04 -07001040 switch (degradation_preference_) {
asaperssonf7e294d2017-06-13 23:25:22 -07001041 case VideoSendStream::DegradationPreference::kBalanced: {
1042 // Try scale up framerate, if higher.
1043 int fps = MaxFps(last_frame_info_->pixel_count());
1044 if (source_proxy_->IncreaseFramerate(fps)) {
1045 GetAdaptCounter().DecrementFramerate(reason, fps);
1046 // Reset framerate in case of fewer fps steps down than up.
1047 if (adapt_counter.FramerateCount() == 0 &&
1048 fps != std::numeric_limits<int>::max()) {
1049 LOG(LS_INFO) << "Removing framerate down-scaling setting.";
1050 source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max());
1051 }
1052 break;
1053 }
1054 // Scale up resolution.
sprang317005a2017-06-08 07:12:17 -07001055 FALLTHROUGH();
asaperssonf7e294d2017-06-13 23:25:22 -07001056 }
asapersson13874762017-06-07 00:01:02 -07001057 case VideoSendStream::DegradationPreference::kMaintainFramerate: {
1058 // Scale up resolution.
1059 int pixel_count = adaptation_request.input_pixel_count_;
1060 if (adapt_counter.ResolutionCount() == 1) {
sprangc5d62e22017-04-02 23:53:04 -07001061 LOG(LS_INFO) << "Removing resolution down-scaling setting.";
asapersson13874762017-06-07 00:01:02 -07001062 pixel_count = std::numeric_limits<int>::max();
sprangc5d62e22017-04-02 23:53:04 -07001063 }
asapersson13874762017-06-07 00:01:02 -07001064 if (!source_proxy_->RequestHigherResolutionThan(pixel_count))
1065 return;
asaperssonf7e294d2017-06-13 23:25:22 -07001066 GetAdaptCounter().DecrementResolution(reason);
sprangc5d62e22017-04-02 23:53:04 -07001067 break;
asapersson13874762017-06-07 00:01:02 -07001068 }
1069 case VideoSendStream::DegradationPreference::kMaintainResolution: {
1070 // Scale up framerate.
1071 int fps = adaptation_request.framerate_fps_;
1072 if (adapt_counter.FramerateCount() == 1) {
sprangc5d62e22017-04-02 23:53:04 -07001073 LOG(LS_INFO) << "Removing framerate down-scaling setting.";
asapersson13874762017-06-07 00:01:02 -07001074 fps = std::numeric_limits<int>::max();
sprangc5d62e22017-04-02 23:53:04 -07001075 }
sprangfda496a2017-06-15 04:21:07 -07001076
1077 const int requested_framerate =
1078 source_proxy_->RequestHigherFramerateThan(fps);
1079 if (requested_framerate == -1) {
1080 overuse_detector_->OnTargetFramerateUpdated(max_framerate_);
asapersson13874762017-06-07 00:01:02 -07001081 return;
sprangfda496a2017-06-15 04:21:07 -07001082 }
1083 overuse_detector_->OnTargetFramerateUpdated(
1084 std::min(max_framerate_, requested_framerate));
asaperssonf7e294d2017-06-13 23:25:22 -07001085 GetAdaptCounter().DecrementFramerate(reason);
sprangc5d62e22017-04-02 23:53:04 -07001086 break;
asapersson13874762017-06-07 00:01:02 -07001087 }
hbos8d609f62017-04-10 07:39:05 -07001088 case VideoSendStream::DegradationPreference::kDegradationDisabled:
asaperssonf7e294d2017-06-13 23:25:22 -07001089 return;
sprangc5d62e22017-04-02 23:53:04 -07001090 }
1091
asaperssond0de2952017-04-21 01:47:31 -07001092 last_adaptation_request_.emplace(adaptation_request);
1093
asapersson09f05612017-05-15 23:40:18 -07001094 UpdateAdaptationStats(reason);
1095
1096 LOG(LS_INFO) << adapt_counter.ToString();
1097}
1098
1099void ViEEncoder::UpdateAdaptationStats(AdaptReason reason) {
asaperssond0de2952017-04-21 01:47:31 -07001100 switch (reason) {
asaperssond0de2952017-04-21 01:47:31 -07001101 case kCpu:
asapersson09f05612017-05-15 23:40:18 -07001102 stats_proxy_->OnCpuAdaptationChanged(GetActiveCounts(kCpu),
1103 GetActiveCounts(kQuality));
1104 break;
1105 case kQuality:
1106 stats_proxy_->OnQualityAdaptationChanged(GetActiveCounts(kCpu),
1107 GetActiveCounts(kQuality));
asaperssond0de2952017-04-21 01:47:31 -07001108 break;
1109 }
perkj26091b12016-09-01 01:17:40 -07001110}
1111
asapersson09f05612017-05-15 23:40:18 -07001112ViEEncoder::AdaptCounts ViEEncoder::GetActiveCounts(AdaptReason reason) {
1113 ViEEncoder::AdaptCounts counts = GetConstAdaptCounter().Counts(reason);
1114 switch (reason) {
1115 case kCpu:
1116 if (!IsFramerateScalingEnabled(degradation_preference_))
1117 counts.fps = -1;
1118 if (!IsResolutionScalingEnabled(degradation_preference_))
1119 counts.resolution = -1;
1120 break;
1121 case kQuality:
1122 if (!IsFramerateScalingEnabled(degradation_preference_) ||
1123 !quality_scaler_) {
1124 counts.fps = -1;
1125 }
1126 if (!IsResolutionScalingEnabled(degradation_preference_) ||
1127 !quality_scaler_) {
1128 counts.resolution = -1;
1129 }
1130 break;
sprangc5d62e22017-04-02 23:53:04 -07001131 }
asapersson09f05612017-05-15 23:40:18 -07001132 return counts;
sprangc5d62e22017-04-02 23:53:04 -07001133}
1134
asapersson09f05612017-05-15 23:40:18 -07001135ViEEncoder::AdaptCounter& ViEEncoder::GetAdaptCounter() {
1136 return adapt_counters_[degradation_preference_];
1137}
1138
1139const ViEEncoder::AdaptCounter& ViEEncoder::GetConstAdaptCounter() {
1140 return adapt_counters_[degradation_preference_];
1141}
1142
1143// Class holding adaptation information.
1144ViEEncoder::AdaptCounter::AdaptCounter() {
1145 fps_counters_.resize(kScaleReasonSize);
1146 resolution_counters_.resize(kScaleReasonSize);
asaperssonf7e294d2017-06-13 23:25:22 -07001147 static_assert(kScaleReasonSize == 2, "Update MoveCount.");
asapersson09f05612017-05-15 23:40:18 -07001148}
1149
1150ViEEncoder::AdaptCounter::~AdaptCounter() {}
1151
1152std::string ViEEncoder::AdaptCounter::ToString() const {
1153 std::stringstream ss;
1154 ss << "Downgrade counts: fps: {" << ToString(fps_counters_);
1155 ss << "}, resolution: {" << ToString(resolution_counters_) << "}";
1156 return ss.str();
1157}
1158
1159ViEEncoder::AdaptCounts ViEEncoder::AdaptCounter::Counts(int reason) const {
1160 AdaptCounts counts;
1161 counts.fps = fps_counters_[reason];
1162 counts.resolution = resolution_counters_[reason];
1163 return counts;
1164}
1165
asaperssonf7e294d2017-06-13 23:25:22 -07001166void ViEEncoder::AdaptCounter::IncrementFramerate(int reason) {
1167 ++(fps_counters_[reason]);
asapersson09f05612017-05-15 23:40:18 -07001168}
1169
asaperssonf7e294d2017-06-13 23:25:22 -07001170void ViEEncoder::AdaptCounter::IncrementResolution(int reason) {
1171 ++(resolution_counters_[reason]);
1172}
1173
1174void ViEEncoder::AdaptCounter::DecrementFramerate(int reason) {
1175 if (fps_counters_[reason] == 0) {
1176 // Balanced mode: Adapt up is in a different order, switch reason.
1177 // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3).
1178 // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0}
1179 // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0}
1180 // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0}
1181 // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0}
1182 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
1183 RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded.";
1184 MoveCount(&resolution_counters_, reason);
1185 MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize);
1186 }
1187 --(fps_counters_[reason]);
1188 RTC_DCHECK_GE(fps_counters_[reason], 0);
1189}
1190
1191void ViEEncoder::AdaptCounter::DecrementResolution(int reason) {
1192 if (resolution_counters_[reason] == 0) {
1193 // Balanced mode: Adapt up is in a different order, switch reason.
1194 RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason.";
1195 RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded.";
1196 MoveCount(&fps_counters_, reason);
1197 MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize);
1198 }
1199 --(resolution_counters_[reason]);
1200 RTC_DCHECK_GE(resolution_counters_[reason], 0);
1201}
1202
1203void ViEEncoder::AdaptCounter::DecrementFramerate(int reason, int cur_fps) {
1204 DecrementFramerate(reason);
1205 // Reset if at max fps (i.e. in case of fewer steps up than down).
1206 if (cur_fps == std::numeric_limits<int>::max())
1207 std::fill(fps_counters_.begin(), fps_counters_.end(), 0);
asapersson09f05612017-05-15 23:40:18 -07001208}
1209
1210int ViEEncoder::AdaptCounter::FramerateCount() const {
1211 return Count(fps_counters_);
1212}
1213
1214int ViEEncoder::AdaptCounter::ResolutionCount() const {
1215 return Count(resolution_counters_);
1216}
1217
asapersson09f05612017-05-15 23:40:18 -07001218int ViEEncoder::AdaptCounter::FramerateCount(int reason) const {
1219 return fps_counters_[reason];
1220}
1221
1222int ViEEncoder::AdaptCounter::ResolutionCount(int reason) const {
1223 return resolution_counters_[reason];
1224}
1225
1226int ViEEncoder::AdaptCounter::TotalCount(int reason) const {
1227 return FramerateCount(reason) + ResolutionCount(reason);
1228}
1229
1230int ViEEncoder::AdaptCounter::Count(const std::vector<int>& counters) const {
1231 return std::accumulate(counters.begin(), counters.end(), 0);
1232}
1233
asaperssonf7e294d2017-06-13 23:25:22 -07001234void ViEEncoder::AdaptCounter::MoveCount(std::vector<int>* counters,
1235 int from_reason) {
1236 int to_reason = (from_reason + 1) % kScaleReasonSize;
1237 ++((*counters)[to_reason]);
1238 --((*counters)[from_reason]);
1239}
1240
asapersson09f05612017-05-15 23:40:18 -07001241std::string ViEEncoder::AdaptCounter::ToString(
1242 const std::vector<int>& counters) const {
1243 std::stringstream ss;
1244 for (size_t reason = 0; reason < kScaleReasonSize; ++reason) {
1245 ss << (reason ? " cpu" : "quality") << ":" << counters[reason];
sprangc5d62e22017-04-02 23:53:04 -07001246 }
asapersson09f05612017-05-15 23:40:18 -07001247 return ss.str();
sprangc5d62e22017-04-02 23:53:04 -07001248}
1249
mflodman@webrtc.org84d17832011-12-01 17:02:23 +00001250} // namespace webrtc