niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
stefan@webrtc.org | 07b45a5 | 2012-02-02 08:37:48 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "video/video_stream_encoder.h" |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 12 | |
stefan@webrtc.org | c3cc375 | 2013-06-04 09:36:56 +0000 | [diff] [blame] | 13 | #include <algorithm> |
perkj | 57c21f9 | 2016-06-17 07:27:16 -0700 | [diff] [blame] | 14 | #include <limits> |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 15 | #include <numeric> |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 16 | #include <utility> |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 17 | |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 18 | #include "absl/memory/memory.h" |
Niels Möller | 4dc66c5 | 2018-10-05 14:17:58 +0200 | [diff] [blame] | 19 | #include "api/video/encoded_image.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 20 | #include "api/video/i420_buffer.h" |
Jiawei Ou | c2ebe21 | 2018-11-08 10:02:56 -0800 | [diff] [blame] | 21 | #include "api/video/video_bitrate_allocator_factory.h" |
Sergey Silkin | 8b9b5f9 | 2018-12-10 09:28:53 +0100 | [diff] [blame] | 22 | #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/video_coding/include/video_codec_initializer.h" |
| 24 | #include "modules/video_coding/include/video_coding.h" |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 25 | #include "modules/video_coding/utility/default_video_bitrate_allocator.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 26 | #include "rtc_base/arraysize.h" |
| 27 | #include "rtc_base/checks.h" |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 28 | #include "rtc_base/experiments/alr_experiment.h" |
Åsa Persson | a945aee | 2018-04-24 16:53:25 +0200 | [diff] [blame] | 29 | #include "rtc_base/experiments/quality_scaling_experiment.h" |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 30 | #include "rtc_base/experiments/rate_control_settings.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 31 | #include "rtc_base/location.h" |
| 32 | #include "rtc_base/logging.h" |
Jonas Olsson | 366a50c | 2018-09-06 13:41:30 +0200 | [diff] [blame] | 33 | #include "rtc_base/strings/string_builder.h" |
Karl Wiberg | 80ba333 | 2018-02-05 10:33:35 +0100 | [diff] [blame] | 34 | #include "rtc_base/system/fallthrough.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 35 | #include "rtc_base/time_utils.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 36 | #include "rtc_base/trace_event.h" |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 37 | #include "system_wrappers/include/field_trial.h" |
nisse | ea3a798 | 2017-05-15 02:42:11 -0700 | [diff] [blame] | 38 | |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 39 | namespace webrtc { |
| 40 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 41 | namespace { |
sprang | b1ca073 | 2017-02-01 08:38:12 -0800 | [diff] [blame] | 42 | |
asapersson | 6ffb67d | 2016-09-12 00:10:45 -0700 | [diff] [blame] | 43 | // Time interval for logging frame counts. |
| 44 | const int64_t kFrameLogIntervalMs = 60000; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 45 | const int kMinFramerateFps = 2; |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 46 | |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 47 | // Time to keep a single cached pending frame in paused state. |
| 48 | const int64_t kPendingFrameTimeoutMs = 1000; |
| 49 | |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 50 | const char kInitialFramedropFieldTrial[] = "WebRTC-InitialFramedrop"; |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 51 | constexpr char kFrameDropperFieldTrial[] = "WebRTC-FrameDropper"; |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 52 | |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 53 | // The maximum number of frames to drop at beginning of stream |
| 54 | // to try and achieve desired bitrate. |
| 55 | const int kMaxInitialFramedrop = 4; |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 56 | // When the first change in BWE above this threshold occurs, |
| 57 | // enable DropFrameDueToSize logic. |
| 58 | const float kFramedropThreshold = 0.3; |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 59 | |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 60 | // Averaging window spanning 90 frames at default 30fps, matching old media |
| 61 | // optimization module defaults. |
| 62 | const int64_t kFrameRateAvergingWindowSizeMs = (1000 / 30) * 90; |
| 63 | |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 64 | const size_t kDefaultPayloadSize = 1440; |
| 65 | |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 66 | // Initial limits for BALANCED degradation preference. |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 67 | int MinFps(int pixels) { |
| 68 | if (pixels <= 320 * 240) { |
| 69 | return 7; |
| 70 | } else if (pixels <= 480 * 270) { |
| 71 | return 10; |
| 72 | } else if (pixels <= 640 * 480) { |
| 73 | return 15; |
| 74 | } else { |
| 75 | return std::numeric_limits<int>::max(); |
| 76 | } |
| 77 | } |
| 78 | |
| 79 | int MaxFps(int pixels) { |
| 80 | if (pixels <= 320 * 240) { |
| 81 | return 10; |
| 82 | } else if (pixels <= 480 * 270) { |
| 83 | return 15; |
| 84 | } else { |
| 85 | return std::numeric_limits<int>::max(); |
| 86 | } |
| 87 | } |
| 88 | |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 89 | uint32_t abs_diff(uint32_t a, uint32_t b) { |
| 90 | return (a < b) ? b - a : a - b; |
| 91 | } |
| 92 | |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 93 | bool IsResolutionScalingEnabled(DegradationPreference degradation_preference) { |
| 94 | return degradation_preference == DegradationPreference::MAINTAIN_FRAMERATE || |
| 95 | degradation_preference == DegradationPreference::BALANCED; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 96 | } |
| 97 | |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 98 | bool IsFramerateScalingEnabled(DegradationPreference degradation_preference) { |
| 99 | return degradation_preference == DegradationPreference::MAINTAIN_RESOLUTION || |
| 100 | degradation_preference == DegradationPreference::BALANCED; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 101 | } |
| 102 | |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 103 | // TODO(pbos): Lower these thresholds (to closer to 100%) when we handle |
| 104 | // pipelining encoders better (multiple input frames before something comes |
| 105 | // out). This should effectively turn off CPU adaptations for systems that |
| 106 | // remotely cope with the load right now. |
| 107 | CpuOveruseOptions GetCpuOveruseOptions( |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 108 | const VideoStreamEncoderSettings& settings, |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 109 | bool full_overuse_time) { |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 110 | CpuOveruseOptions options; |
| 111 | |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 112 | if (full_overuse_time) { |
Niels Möller | d1f7eb6 | 2018-03-28 16:40:58 +0200 | [diff] [blame] | 113 | options.low_encode_usage_threshold_percent = 150; |
| 114 | options.high_encode_usage_threshold_percent = 200; |
| 115 | } |
| 116 | if (settings.experiment_cpu_load_estimator) { |
| 117 | options.filter_time_ms = 5 * rtc::kNumMillisecsPerSec; |
| 118 | } |
| 119 | |
| 120 | return options; |
| 121 | } |
| 122 | |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 123 | bool RequiresEncoderReset(const VideoCodec& previous_send_codec, |
| 124 | const VideoCodec& new_send_codec) { |
| 125 | // Does not check startBitrate or maxFramerate. |
| 126 | if (new_send_codec.codecType != previous_send_codec.codecType || |
| 127 | new_send_codec.width != previous_send_codec.width || |
| 128 | new_send_codec.height != previous_send_codec.height || |
| 129 | new_send_codec.maxBitrate != previous_send_codec.maxBitrate || |
| 130 | new_send_codec.minBitrate != previous_send_codec.minBitrate || |
| 131 | new_send_codec.qpMax != previous_send_codec.qpMax || |
| 132 | new_send_codec.numberOfSimulcastStreams != |
| 133 | previous_send_codec.numberOfSimulcastStreams || |
| 134 | new_send_codec.mode != previous_send_codec.mode) { |
| 135 | return true; |
| 136 | } |
| 137 | |
| 138 | switch (new_send_codec.codecType) { |
| 139 | case kVideoCodecVP8: |
| 140 | if (new_send_codec.VP8() != previous_send_codec.VP8()) { |
| 141 | return true; |
| 142 | } |
| 143 | break; |
| 144 | |
| 145 | case kVideoCodecVP9: |
| 146 | if (new_send_codec.VP9() != previous_send_codec.VP9()) { |
| 147 | return true; |
| 148 | } |
| 149 | break; |
| 150 | |
| 151 | case kVideoCodecH264: |
| 152 | if (new_send_codec.H264() != previous_send_codec.H264()) { |
| 153 | return true; |
| 154 | } |
| 155 | break; |
| 156 | |
| 157 | default: |
| 158 | break; |
| 159 | } |
| 160 | |
| 161 | for (unsigned char i = 0; i < new_send_codec.numberOfSimulcastStreams; ++i) { |
| 162 | if (new_send_codec.simulcastStream[i] != |
| 163 | previous_send_codec.simulcastStream[i]) |
| 164 | return true; |
| 165 | } |
| 166 | return false; |
| 167 | } |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 168 | |
| 169 | std::array<uint8_t, 2> GetExperimentGroups() { |
| 170 | std::array<uint8_t, 2> experiment_groups; |
| 171 | absl::optional<AlrExperimentSettings> experiment_settings = |
| 172 | AlrExperimentSettings::CreateFromFieldTrial( |
| 173 | AlrExperimentSettings::kStrictPacingAndProbingExperimentName); |
| 174 | if (experiment_settings) { |
| 175 | experiment_groups[0] = experiment_settings->group_id + 1; |
| 176 | } else { |
| 177 | experiment_groups[0] = 0; |
| 178 | } |
| 179 | experiment_settings = AlrExperimentSettings::CreateFromFieldTrial( |
| 180 | AlrExperimentSettings::kScreenshareProbingBweExperimentName); |
| 181 | if (experiment_settings) { |
| 182 | experiment_groups[1] = experiment_settings->group_id + 1; |
| 183 | } else { |
| 184 | experiment_groups[1] = 0; |
| 185 | } |
| 186 | return experiment_groups; |
| 187 | } |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 188 | } // namespace |
| 189 | |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 190 | // VideoSourceProxy is responsible ensuring thread safety between calls to |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 191 | // VideoStreamEncoder::SetSource that will happen on libjingle's worker thread |
| 192 | // when a video capturer is connected to the encoder and the encoder task queue |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 193 | // (encoder_queue_) where the encoder reports its VideoSinkWants. |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 194 | class VideoStreamEncoder::VideoSourceProxy { |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 195 | public: |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 196 | explicit VideoSourceProxy(VideoStreamEncoder* video_stream_encoder) |
| 197 | : video_stream_encoder_(video_stream_encoder), |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 198 | degradation_preference_(DegradationPreference::DISABLED), |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 199 | source_(nullptr), |
| 200 | max_framerate_(std::numeric_limits<int>::max()) {} |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 201 | |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 202 | void SetSource(rtc::VideoSourceInterface<VideoFrame>* source, |
| 203 | const DegradationPreference& degradation_preference) { |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 204 | // Called on libjingle's worker thread. |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 205 | RTC_DCHECK_CALLED_SEQUENTIALLY(&main_checker_); |
| 206 | rtc::VideoSourceInterface<VideoFrame>* old_source = nullptr; |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 207 | rtc::VideoSinkWants wants; |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 208 | { |
| 209 | rtc::CritScope lock(&crit_); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 210 | degradation_preference_ = degradation_preference; |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 211 | old_source = source_; |
| 212 | source_ = source; |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 213 | wants = GetActiveSinkWantsInternal(); |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 214 | } |
| 215 | |
| 216 | if (old_source != source && old_source != nullptr) { |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 217 | old_source->RemoveSink(video_stream_encoder_); |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 218 | } |
| 219 | |
| 220 | if (!source) { |
| 221 | return; |
| 222 | } |
| 223 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 224 | source->AddOrUpdateSink(video_stream_encoder_, wants); |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 225 | } |
| 226 | |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 227 | void SetMaxFramerate(int max_framerate) { |
| 228 | RTC_DCHECK_GT(max_framerate, 0); |
| 229 | rtc::CritScope lock(&crit_); |
| 230 | if (max_framerate == max_framerate_) |
| 231 | return; |
| 232 | |
| 233 | RTC_LOG(LS_INFO) << "Set max framerate: " << max_framerate; |
| 234 | max_framerate_ = max_framerate; |
| 235 | if (source_) { |
| 236 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 237 | GetActiveSinkWantsInternal()); |
| 238 | } |
| 239 | } |
| 240 | |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 241 | void SetWantsRotationApplied(bool rotation_applied) { |
| 242 | rtc::CritScope lock(&crit_); |
| 243 | sink_wants_.rotation_applied = rotation_applied; |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 244 | if (source_) { |
| 245 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 246 | GetActiveSinkWantsInternal()); |
| 247 | } |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 248 | } |
| 249 | |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 250 | rtc::VideoSinkWants GetActiveSinkWants() { |
| 251 | rtc::CritScope lock(&crit_); |
| 252 | return GetActiveSinkWantsInternal(); |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 253 | } |
| 254 | |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 255 | void ResetPixelFpsCount() { |
| 256 | rtc::CritScope lock(&crit_); |
| 257 | sink_wants_.max_pixel_count = std::numeric_limits<int>::max(); |
| 258 | sink_wants_.target_pixel_count.reset(); |
| 259 | sink_wants_.max_framerate_fps = std::numeric_limits<int>::max(); |
| 260 | if (source_) |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 261 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 262 | GetActiveSinkWantsInternal()); |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 263 | } |
| 264 | |
Åsa Persson | c3ed630 | 2017-11-16 14:04:52 +0100 | [diff] [blame] | 265 | bool RequestResolutionLowerThan(int pixel_count, |
| 266 | int min_pixels_per_frame, |
| 267 | bool* min_pixels_reached) { |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 268 | // Called on the encoder task queue. |
| 269 | rtc::CritScope lock(&crit_); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 270 | if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { |
asapersson | 02465b8 | 2017-04-10 01:12:52 -0700 | [diff] [blame] | 271 | // This can happen since |degradation_preference_| is set on libjingle's |
| 272 | // worker thread but the adaptation is done on the encoder task queue. |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 273 | return false; |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 274 | } |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 275 | // The input video frame size will have a resolution less than or equal to |
| 276 | // |max_pixel_count| depending on how the source can scale the frame size. |
kthelgason | 5e13d41 | 2016-12-01 03:59:51 -0800 | [diff] [blame] | 277 | const int pixels_wanted = (pixel_count * 3) / 5; |
Åsa Persson | c3ed630 | 2017-11-16 14:04:52 +0100 | [diff] [blame] | 278 | if (pixels_wanted >= sink_wants_.max_pixel_count) { |
| 279 | return false; |
| 280 | } |
| 281 | if (pixels_wanted < min_pixels_per_frame) { |
| 282 | *min_pixels_reached = true; |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 283 | return false; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 284 | } |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 285 | RTC_LOG(LS_INFO) << "Scaling down resolution, max pixels: " |
| 286 | << pixels_wanted; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 287 | sink_wants_.max_pixel_count = pixels_wanted; |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 288 | sink_wants_.target_pixel_count = absl::nullopt; |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 289 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 290 | GetActiveSinkWantsInternal()); |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 291 | return true; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 292 | } |
| 293 | |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 294 | int RequestFramerateLowerThan(int fps) { |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 295 | // Called on the encoder task queue. |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 296 | // The input video frame rate will be scaled down to 2/3, rounding down. |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 297 | int framerate_wanted = (fps * 2) / 3; |
| 298 | return RestrictFramerate(framerate_wanted) ? framerate_wanted : -1; |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 299 | } |
| 300 | |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 301 | bool RequestHigherResolutionThan(int pixel_count) { |
| 302 | // Called on the encoder task queue. |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 303 | rtc::CritScope lock(&crit_); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 304 | if (!source_ || !IsResolutionScalingEnabled(degradation_preference_)) { |
asapersson | 02465b8 | 2017-04-10 01:12:52 -0700 | [diff] [blame] | 305 | // This can happen since |degradation_preference_| is set on libjingle's |
| 306 | // worker thread but the adaptation is done on the encoder task queue. |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 307 | return false; |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 308 | } |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 309 | int max_pixels_wanted = pixel_count; |
| 310 | if (max_pixels_wanted != std::numeric_limits<int>::max()) |
| 311 | max_pixels_wanted = pixel_count * 4; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 312 | |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 313 | if (max_pixels_wanted <= sink_wants_.max_pixel_count) |
| 314 | return false; |
| 315 | |
| 316 | sink_wants_.max_pixel_count = max_pixels_wanted; |
| 317 | if (max_pixels_wanted == std::numeric_limits<int>::max()) { |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 318 | // Remove any constraints. |
| 319 | sink_wants_.target_pixel_count.reset(); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 320 | } else { |
| 321 | // On step down we request at most 3/5 the pixel count of the previous |
| 322 | // resolution, so in order to take "one step up" we request a resolution |
| 323 | // as close as possible to 5/3 of the current resolution. The actual pixel |
| 324 | // count selected depends on the capabilities of the source. In order to |
| 325 | // not take a too large step up, we cap the requested pixel count to be at |
| 326 | // most four time the current number of pixels. |
Oskar Sundbom | 8e07c13 | 2018-01-08 16:45:42 +0100 | [diff] [blame] | 327 | sink_wants_.target_pixel_count = (pixel_count * 5) / 3; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 328 | } |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 329 | RTC_LOG(LS_INFO) << "Scaling up resolution, max pixels: " |
| 330 | << max_pixels_wanted; |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 331 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 332 | GetActiveSinkWantsInternal()); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 333 | return true; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 334 | } |
| 335 | |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 336 | // Request upgrade in framerate. Returns the new requested frame, or -1 if |
| 337 | // no change requested. Note that maxint may be returned if limits due to |
| 338 | // adaptation requests are removed completely. In that case, consider |
| 339 | // |max_framerate_| to be the current limit (assuming the capturer complies). |
| 340 | int RequestHigherFramerateThan(int fps) { |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 341 | // Called on the encoder task queue. |
| 342 | // The input frame rate will be scaled up to the last step, with rounding. |
| 343 | int framerate_wanted = fps; |
| 344 | if (fps != std::numeric_limits<int>::max()) |
| 345 | framerate_wanted = (fps * 3) / 2; |
| 346 | |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 347 | return IncreaseFramerate(framerate_wanted) ? framerate_wanted : -1; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 348 | } |
| 349 | |
| 350 | bool RestrictFramerate(int fps) { |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 351 | // Called on the encoder task queue. |
| 352 | rtc::CritScope lock(&crit_); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 353 | if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) |
| 354 | return false; |
| 355 | |
| 356 | const int fps_wanted = std::max(kMinFramerateFps, fps); |
| 357 | if (fps_wanted >= sink_wants_.max_framerate_fps) |
| 358 | return false; |
| 359 | |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 360 | RTC_LOG(LS_INFO) << "Scaling down framerate: " << fps_wanted; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 361 | sink_wants_.max_framerate_fps = fps_wanted; |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 362 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 363 | GetActiveSinkWantsInternal()); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 364 | return true; |
| 365 | } |
| 366 | |
| 367 | bool IncreaseFramerate(int fps) { |
| 368 | // Called on the encoder task queue. |
| 369 | rtc::CritScope lock(&crit_); |
| 370 | if (!source_ || !IsFramerateScalingEnabled(degradation_preference_)) |
| 371 | return false; |
| 372 | |
| 373 | const int fps_wanted = std::max(kMinFramerateFps, fps); |
| 374 | if (fps_wanted <= sink_wants_.max_framerate_fps) |
| 375 | return false; |
| 376 | |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 377 | RTC_LOG(LS_INFO) << "Scaling up framerate: " << fps_wanted; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 378 | sink_wants_.max_framerate_fps = fps_wanted; |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 379 | source_->AddOrUpdateSink(video_stream_encoder_, |
| 380 | GetActiveSinkWantsInternal()); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 381 | return true; |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 382 | } |
| 383 | |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 384 | private: |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 385 | rtc::VideoSinkWants GetActiveSinkWantsInternal() |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 386 | RTC_EXCLUSIVE_LOCKS_REQUIRED(&crit_) { |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 387 | rtc::VideoSinkWants wants = sink_wants_; |
| 388 | // Clear any constraints from the current sink wants that don't apply to |
| 389 | // the used degradation_preference. |
| 390 | switch (degradation_preference_) { |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 391 | case DegradationPreference::BALANCED: |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 392 | break; |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 393 | case DegradationPreference::MAINTAIN_FRAMERATE: |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 394 | wants.max_framerate_fps = std::numeric_limits<int>::max(); |
| 395 | break; |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 396 | case DegradationPreference::MAINTAIN_RESOLUTION: |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 397 | wants.max_pixel_count = std::numeric_limits<int>::max(); |
| 398 | wants.target_pixel_count.reset(); |
| 399 | break; |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 400 | case DegradationPreference::DISABLED: |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 401 | wants.max_pixel_count = std::numeric_limits<int>::max(); |
| 402 | wants.target_pixel_count.reset(); |
| 403 | wants.max_framerate_fps = std::numeric_limits<int>::max(); |
| 404 | } |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 405 | // Limit to configured max framerate. |
| 406 | wants.max_framerate_fps = std::min(max_framerate_, wants.max_framerate_fps); |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 407 | return wants; |
| 408 | } |
| 409 | |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 410 | rtc::CriticalSection crit_; |
| 411 | rtc::SequencedTaskChecker main_checker_; |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 412 | VideoStreamEncoder* const video_stream_encoder_; |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 413 | rtc::VideoSinkWants sink_wants_ RTC_GUARDED_BY(&crit_); |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 414 | DegradationPreference degradation_preference_ RTC_GUARDED_BY(&crit_); |
danilchap | a37de39 | 2017-09-09 04:17:22 -0700 | [diff] [blame] | 415 | rtc::VideoSourceInterface<VideoFrame>* source_ RTC_GUARDED_BY(&crit_); |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 416 | int max_framerate_ RTC_GUARDED_BY(&crit_); |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 417 | |
| 418 | RTC_DISALLOW_COPY_AND_ASSIGN(VideoSourceProxy); |
| 419 | }; |
| 420 | |
Åsa Persson | 0122e84 | 2017-10-16 12:19:23 +0200 | [diff] [blame] | 421 | VideoStreamEncoder::VideoStreamEncoder( |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 422 | Clock* clock, |
Åsa Persson | 0122e84 | 2017-10-16 12:19:23 +0200 | [diff] [blame] | 423 | uint32_t number_of_cores, |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 424 | VideoStreamEncoderObserver* encoder_stats_observer, |
| 425 | const VideoStreamEncoderSettings& settings, |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 426 | std::unique_ptr<OveruseFrameDetector> overuse_detector, |
| 427 | TaskQueueFactory* task_queue_factory) |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 428 | : shutdown_event_(true /* manual_reset */, false), |
| 429 | number_of_cores_(number_of_cores), |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 430 | initial_framedrop_(0), |
| 431 | initial_framedrop_on_bwe_enabled_( |
| 432 | webrtc::field_trial::IsEnabled(kInitialFramedropFieldTrial)), |
Åsa Persson | a945aee | 2018-04-24 16:53:25 +0200 | [diff] [blame] | 433 | quality_scaling_experiment_enabled_(QualityScalingExperiment::Enabled()), |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 434 | source_proxy_(new VideoSourceProxy(this)), |
Per | a48ddb7 | 2016-09-29 11:48:50 +0200 | [diff] [blame] | 435 | sink_(nullptr), |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 436 | settings_(settings), |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 437 | rate_control_settings_(RateControlSettings::ParseFromFieldTrials()), |
Niels Möller | 73f29cb | 2018-01-31 16:09:31 +0100 | [diff] [blame] | 438 | overuse_detector_(std::move(overuse_detector)), |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 439 | encoder_stats_observer_(encoder_stats_observer), |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 440 | encoder_initialized_(false), |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 441 | max_framerate_(-1), |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 442 | pending_encoder_reconfiguration_(false), |
Mirta Dvornicic | 1ec2a16 | 2018-12-10 09:47:34 +0000 | [diff] [blame] | 443 | pending_encoder_creation_(false), |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 444 | crop_width_(0), |
| 445 | crop_height_(0), |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 446 | encoder_start_bitrate_bps_(0), |
Per | a48ddb7 | 2016-09-29 11:48:50 +0200 | [diff] [blame] | 447 | max_data_payload_length_(0), |
pbos@webrtc.org | 143451d | 2015-03-18 14:40:03 +0000 | [diff] [blame] | 448 | last_observed_bitrate_bps_(0), |
stefan@webrtc.org | 792f1a1 | 2015-03-04 12:24:26 +0000 | [diff] [blame] | 449 | encoder_paused_and_dropped_frame_(false), |
Sebastian Jansson | 572c60f | 2019-03-04 18:30:41 +0100 | [diff] [blame] | 450 | clock_(clock), |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 451 | degradation_preference_(DegradationPreference::DISABLED), |
Yuwei Huang | d9f99c1 | 2017-10-24 15:40:52 -0700 | [diff] [blame] | 452 | posted_frames_waiting_for_encode_(0), |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 453 | last_captured_timestamp_(0), |
| 454 | delta_ntp_internal_ms_(clock_->CurrentNtpInMilliseconds() - |
| 455 | clock_->TimeInMilliseconds()), |
asapersson | 6ffb67d | 2016-09-12 00:10:45 -0700 | [diff] [blame] | 456 | last_frame_log_ms_(clock_->TimeInMilliseconds()), |
| 457 | captured_frame_count_(0), |
| 458 | dropped_frame_count_(0), |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 459 | pending_frame_post_time_us_(0), |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 460 | accumulated_update_rect_{0, 0, 0, 0}, |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 461 | bitrate_observer_(nullptr), |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 462 | last_framerate_fps_(0), |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 463 | force_disable_frame_dropper_(false), |
| 464 | input_framerate_(kFrameRateAvergingWindowSizeMs, 1000), |
| 465 | pending_frame_drops_(0), |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 466 | next_frame_types_(1, kVideoFrameDelta), |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 467 | frame_encoder_timer_(this), |
| 468 | experiment_groups_(GetExperimentGroups()), |
Sebastian Jansson | 74682c1 | 2019-03-01 11:50:20 +0100 | [diff] [blame] | 469 | encoder_queue_(task_queue_factory->CreateTaskQueue( |
| 470 | "EncoderQueue", |
| 471 | TaskQueueFactory::Priority::NORMAL)) { |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 472 | RTC_DCHECK(encoder_stats_observer); |
Niels Möller | 73f29cb | 2018-01-31 16:09:31 +0100 | [diff] [blame] | 473 | RTC_DCHECK(overuse_detector_); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 474 | RTC_DCHECK_GE(number_of_cores, 1); |
mflodman@webrtc.org | 02270cd | 2015-02-06 13:10:19 +0000 | [diff] [blame] | 475 | } |
| 476 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 477 | VideoStreamEncoder::~VideoStreamEncoder() { |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 478 | RTC_DCHECK_RUN_ON(&thread_checker_); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 479 | RTC_DCHECK(shutdown_event_.Wait(0)) |
| 480 | << "Must call ::Stop() before destruction."; |
| 481 | } |
| 482 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 483 | void VideoStreamEncoder::Stop() { |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 484 | RTC_DCHECK_RUN_ON(&thread_checker_); |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 485 | source_proxy_->SetSource(nullptr, DegradationPreference()); |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 486 | encoder_queue_.PostTask([this] { |
| 487 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 488 | overuse_detector_->StopCheckForOveruse(); |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 489 | rate_allocator_ = nullptr; |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 490 | bitrate_observer_ = nullptr; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 491 | ReleaseEncoder(); |
kthelgason | 876222f | 2016-11-29 01:44:11 -0800 | [diff] [blame] | 492 | quality_scaler_ = nullptr; |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 493 | shutdown_event_.Set(); |
| 494 | }); |
| 495 | |
| 496 | shutdown_event_.Wait(rtc::Event::kForever); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 497 | } |
| 498 | |
Niels Möller | 0327c2d | 2018-05-21 14:09:31 +0200 | [diff] [blame] | 499 | void VideoStreamEncoder::SetBitrateAllocationObserver( |
sprang | 1a646ee | 2016-12-01 06:34:11 -0800 | [diff] [blame] | 500 | VideoBitrateAllocationObserver* bitrate_observer) { |
| 501 | RTC_DCHECK_RUN_ON(&thread_checker_); |
| 502 | encoder_queue_.PostTask([this, bitrate_observer] { |
| 503 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 504 | RTC_DCHECK(!bitrate_observer_); |
| 505 | bitrate_observer_ = bitrate_observer; |
| 506 | }); |
| 507 | } |
| 508 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 509 | void VideoStreamEncoder::SetSource( |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 510 | rtc::VideoSourceInterface<VideoFrame>* source, |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 511 | const DegradationPreference& degradation_preference) { |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 512 | RTC_DCHECK_RUN_ON(&thread_checker_); |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 513 | source_proxy_->SetSource(source, degradation_preference); |
| 514 | encoder_queue_.PostTask([this, degradation_preference] { |
| 515 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 516 | if (degradation_preference_ != degradation_preference) { |
| 517 | // Reset adaptation state, so that we're not tricked into thinking there's |
| 518 | // an already pending request of the same type. |
| 519 | last_adaptation_request_.reset(); |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 520 | if (degradation_preference == DegradationPreference::BALANCED || |
| 521 | degradation_preference_ == DegradationPreference::BALANCED) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 522 | // TODO(asapersson): Consider removing |adapt_counters_| map and use one |
| 523 | // AdaptCounter for all modes. |
| 524 | source_proxy_->ResetPixelFpsCount(); |
| 525 | adapt_counters_.clear(); |
| 526 | } |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 527 | } |
sprang | b1ca073 | 2017-02-01 08:38:12 -0800 | [diff] [blame] | 528 | degradation_preference_ = degradation_preference; |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 529 | |
Mirta Dvornicic | 1ec2a16 | 2018-12-10 09:47:34 +0000 | [diff] [blame] | 530 | if (encoder_) |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 531 | ConfigureQualityScaler(encoder_->GetEncoderInfo()); |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 532 | |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 533 | if (!IsFramerateScalingEnabled(degradation_preference) && |
| 534 | max_framerate_ != -1) { |
| 535 | // If frame rate scaling is no longer allowed, remove any potential |
| 536 | // allowance for longer frame intervals. |
| 537 | overuse_detector_->OnTargetFramerateUpdated(max_framerate_); |
| 538 | } |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 539 | }); |
perkj | a49cbd3 | 2016-09-16 07:53:41 -0700 | [diff] [blame] | 540 | } |
| 541 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 542 | void VideoStreamEncoder::SetSink(EncoderSink* sink, bool rotation_applied) { |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 543 | source_proxy_->SetWantsRotationApplied(rotation_applied); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 544 | encoder_queue_.PostTask([this, sink] { |
| 545 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 546 | sink_ = sink; |
| 547 | }); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 548 | } |
| 549 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 550 | void VideoStreamEncoder::SetStartBitrate(int start_bitrate_bps) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 551 | encoder_queue_.PostTask([this, start_bitrate_bps] { |
| 552 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 553 | encoder_start_bitrate_bps_ = start_bitrate_bps; |
| 554 | }); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 555 | } |
Peter Boström | 00b9d21 | 2016-05-19 16:59:03 +0200 | [diff] [blame] | 556 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 557 | void VideoStreamEncoder::ConfigureEncoder(VideoEncoderConfig config, |
Niels Möller | f133856 | 2018-04-26 09:51:47 +0200 | [diff] [blame] | 558 | size_t max_data_payload_length) { |
Sebastian Jansson | 3dc0125 | 2018-03-19 19:27:44 +0100 | [diff] [blame] | 559 | // TODO(srte): This struct should be replaced by a lambda with move capture |
| 560 | // when C++14 lambda is allowed. |
| 561 | struct ConfigureEncoderTask { |
| 562 | void operator()() { |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 563 | encoder->ConfigureEncoderOnTaskQueue(std::move(config), |
| 564 | max_data_payload_length); |
Sebastian Jansson | 3dc0125 | 2018-03-19 19:27:44 +0100 | [diff] [blame] | 565 | } |
| 566 | VideoStreamEncoder* encoder; |
| 567 | VideoEncoderConfig config; |
| 568 | size_t max_data_payload_length; |
Sebastian Jansson | 3dc0125 | 2018-03-19 19:27:44 +0100 | [diff] [blame] | 569 | }; |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 570 | encoder_queue_.PostTask( |
| 571 | ConfigureEncoderTask{this, std::move(config), max_data_payload_length}); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 572 | } |
| 573 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 574 | void VideoStreamEncoder::ConfigureEncoderOnTaskQueue( |
| 575 | VideoEncoderConfig config, |
Niels Möller | f133856 | 2018-04-26 09:51:47 +0200 | [diff] [blame] | 576 | size_t max_data_payload_length) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 577 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 578 | RTC_DCHECK(sink_); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 579 | RTC_LOG(LS_INFO) << "ConfigureEncoder requested."; |
Per | a48ddb7 | 2016-09-29 11:48:50 +0200 | [diff] [blame] | 580 | |
Mirta Dvornicic | 1ec2a16 | 2018-12-10 09:47:34 +0000 | [diff] [blame] | 581 | pending_encoder_creation_ = |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 582 | (!encoder_ || encoder_config_.video_format != config.video_format || |
| 583 | max_data_payload_length_ != max_data_payload_length); |
Per | a48ddb7 | 2016-09-29 11:48:50 +0200 | [diff] [blame] | 584 | encoder_config_ = std::move(config); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 585 | max_data_payload_length_ = max_data_payload_length; |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 586 | pending_encoder_reconfiguration_ = true; |
Per | a48ddb7 | 2016-09-29 11:48:50 +0200 | [diff] [blame] | 587 | |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 588 | // Reconfigure the encoder now if the encoder has an internal source or |
Per | 21d45d2 | 2016-10-30 21:37:57 +0100 | [diff] [blame] | 589 | // if the frame resolution is known. Otherwise, the reconfiguration is |
| 590 | // deferred until the next frame to minimize the number of reconfigurations. |
| 591 | // The codec configuration depends on incoming video frame size. |
| 592 | if (last_frame_info_) { |
| 593 | ReconfigureEncoder(); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 594 | } else { |
| 595 | codec_info_ = settings_.encoder_factory->QueryVideoEncoder( |
| 596 | encoder_config_.video_format); |
| 597 | if (HasInternalSource()) { |
| 598 | last_frame_info_ = VideoFrameInfo(176, 144, false); |
| 599 | ReconfigureEncoder(); |
| 600 | } |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 601 | } |
| 602 | } |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 603 | |
Seth Hampson | cc7125f | 2018-02-02 08:46:16 -0800 | [diff] [blame] | 604 | // TODO(bugs.webrtc.org/8807): Currently this always does a hard |
| 605 | // reconfiguration, but this isn't always necessary. Add in logic to only update |
| 606 | // the VideoBitrateAllocator and call OnEncoderConfigurationChanged with a |
| 607 | // "soft" reconfiguration. |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 608 | void VideoStreamEncoder::ReconfigureEncoder() { |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 609 | RTC_DCHECK(pending_encoder_reconfiguration_); |
| 610 | std::vector<VideoStream> streams = |
| 611 | encoder_config_.video_stream_factory->CreateEncoderStreams( |
| 612 | last_frame_info_->width, last_frame_info_->height, encoder_config_); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 613 | |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 614 | // TODO(ilnik): If configured resolution is significantly less than provided, |
| 615 | // e.g. because there are not enough SSRCs for all simulcast streams, |
| 616 | // signal new resolutions via SinkWants to video source. |
| 617 | |
| 618 | // Stream dimensions may be not equal to given because of a simulcast |
| 619 | // restrictions. |
Florent Castelli | 450b548 | 2018-11-29 17:32:47 +0100 | [diff] [blame] | 620 | auto highest_stream = std::max_element( |
| 621 | streams.begin(), streams.end(), |
| 622 | [](const webrtc::VideoStream& a, const webrtc::VideoStream& b) { |
| 623 | return std::tie(a.width, a.height) < std::tie(b.width, b.height); |
| 624 | }); |
| 625 | int highest_stream_width = static_cast<int>(highest_stream->width); |
| 626 | int highest_stream_height = static_cast<int>(highest_stream->height); |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 627 | // Dimension may be reduced to be, e.g. divisible by 4. |
| 628 | RTC_CHECK_GE(last_frame_info_->width, highest_stream_width); |
| 629 | RTC_CHECK_GE(last_frame_info_->height, highest_stream_height); |
| 630 | crop_width_ = last_frame_info_->width - highest_stream_width; |
| 631 | crop_height_ = last_frame_info_->height - highest_stream_height; |
| 632 | |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 633 | VideoCodec codec; |
Jiawei Ou | c2ebe21 | 2018-11-08 10:02:56 -0800 | [diff] [blame] | 634 | if (!VideoCodecInitializer::SetupCodec(encoder_config_, streams, &codec)) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 635 | RTC_LOG(LS_ERROR) << "Failed to create encoder configuration."; |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 636 | } |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 637 | |
Jiawei Ou | c2ebe21 | 2018-11-08 10:02:56 -0800 | [diff] [blame] | 638 | rate_allocator_ = |
| 639 | settings_.bitrate_allocator_factory->CreateVideoBitrateAllocator(codec); |
Jiawei Ou | c2ebe21 | 2018-11-08 10:02:56 -0800 | [diff] [blame] | 640 | |
“Michael | 277a656 | 2018-06-01 14:09:19 -0500 | [diff] [blame] | 641 | // Set min_bitrate_bps, max_bitrate_bps, and max padding bit rate for VP9. |
| 642 | if (encoder_config_.codec_type == kVideoCodecVP9) { |
“Michael | 277a656 | 2018-06-01 14:09:19 -0500 | [diff] [blame] | 643 | // Lower max bitrate to the level codec actually can produce. |
Sergey Silkin | 8b9b5f9 | 2018-12-10 09:28:53 +0100 | [diff] [blame] | 644 | streams[0].max_bitrate_bps = std::min<int>( |
| 645 | streams[0].max_bitrate_bps, SvcRateAllocator::GetMaxBitrateBps(codec)); |
“Michael | 277a656 | 2018-06-01 14:09:19 -0500 | [diff] [blame] | 646 | streams[0].min_bitrate_bps = codec.spatialLayers[0].minBitrate * 1000; |
Sergey Silkin | 8b9b5f9 | 2018-12-10 09:28:53 +0100 | [diff] [blame] | 647 | // target_bitrate_bps specifies the maximum padding bitrate. |
“Michael | 277a656 | 2018-06-01 14:09:19 -0500 | [diff] [blame] | 648 | streams[0].target_bitrate_bps = |
Sergey Silkin | 8b9b5f9 | 2018-12-10 09:28:53 +0100 | [diff] [blame] | 649 | SvcRateAllocator::GetPaddingBitrateBps(codec); |
“Michael | 277a656 | 2018-06-01 14:09:19 -0500 | [diff] [blame] | 650 | } |
| 651 | |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 652 | codec.startBitrate = |
| 653 | std::max(encoder_start_bitrate_bps_ / 1000, codec.minBitrate); |
| 654 | codec.startBitrate = std::min(codec.startBitrate, codec.maxBitrate); |
| 655 | codec.expect_encode_from_texture = last_frame_info_->is_texture; |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 656 | // Make sure the start bit rate is sane... |
| 657 | RTC_DCHECK_LE(codec.startBitrate, 1000000); |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 658 | max_framerate_ = codec.maxFramerate; |
Åsa Persson | 8c1bf95 | 2018-09-13 10:42:19 +0200 | [diff] [blame] | 659 | |
| 660 | // Inform source about max configured framerate. |
| 661 | int max_framerate = 0; |
| 662 | for (const auto& stream : streams) { |
| 663 | max_framerate = std::max(stream.max_framerate, max_framerate); |
| 664 | } |
| 665 | source_proxy_->SetMaxFramerate(max_framerate); |
Stefan Holmer | e590416 | 2015-03-26 11:11:06 +0100 | [diff] [blame] | 666 | |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 667 | if (codec.maxBitrate == 0) { |
| 668 | // max is one bit per pixel |
| 669 | codec.maxBitrate = |
| 670 | (static_cast<int>(codec.height) * static_cast<int>(codec.width) * |
| 671 | static_cast<int>(codec.maxFramerate)) / |
| 672 | 1000; |
| 673 | if (codec.startBitrate > codec.maxBitrate) { |
| 674 | // But if the user tries to set a higher start bit rate we will |
| 675 | // increase the max accordingly. |
| 676 | codec.maxBitrate = codec.startBitrate; |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | if (codec.startBitrate > codec.maxBitrate) { |
| 681 | codec.startBitrate = codec.maxBitrate; |
| 682 | } |
| 683 | |
| 684 | // Reset (release existing encoder) if one exists and anything except |
| 685 | // start bitrate or max framerate has changed. Don't call Release() if |
| 686 | // |pending_encoder_creation_| as that means this is a new encoder |
| 687 | // that has not yet been initialized. |
| 688 | const bool reset_required = RequiresEncoderReset(codec, send_codec_); |
| 689 | send_codec_ = codec; |
| 690 | |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 691 | // Keep the same encoder, as long as the video_format is unchanged. |
Mirta Dvornicic | ccc1b57 | 2019-01-15 12:42:18 +0100 | [diff] [blame] | 692 | // Encoder creation block is split in two since EncoderInfo needed to start |
| 693 | // CPU adaptation with the correct settings should be polled after |
| 694 | // encoder_->InitEncode(). |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 695 | bool success = true; |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 696 | if (pending_encoder_creation_ || reset_required) { |
| 697 | ReleaseEncoder(); |
| 698 | if (pending_encoder_creation_) { |
| 699 | encoder_ = settings_.encoder_factory->CreateVideoEncoder( |
| 700 | encoder_config_.video_format); |
| 701 | // TODO(nisse): What to do if creating the encoder fails? Crash, |
| 702 | // or just discard incoming frames? |
| 703 | RTC_CHECK(encoder_); |
| 704 | codec_info_ = settings_.encoder_factory->QueryVideoEncoder( |
| 705 | encoder_config_.video_format); |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 706 | } |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 707 | |
| 708 | if (encoder_->InitEncode(&send_codec_, number_of_cores_, |
| 709 | max_data_payload_length_ > 0 |
| 710 | ? max_data_payload_length_ |
| 711 | : kDefaultPayloadSize) != 0) { |
| 712 | RTC_LOG(LS_ERROR) << "Failed to initialize the encoder associated with " |
| 713 | "codec type: " |
| 714 | << CodecTypeToPayloadString(send_codec_.codecType) |
| 715 | << " (" << send_codec_.codecType << ")"; |
| 716 | ReleaseEncoder(); |
| 717 | success = false; |
| 718 | } else { |
| 719 | encoder_initialized_ = true; |
| 720 | encoder_->RegisterEncodeCompleteCallback(this); |
| 721 | frame_encoder_timer_.OnEncoderInit(send_codec_, HasInternalSource()); |
| 722 | } |
| 723 | |
| 724 | frame_encoder_timer_.Reset(); |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 725 | } |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 726 | |
| 727 | if (success) { |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 728 | next_frame_types_.clear(); |
| 729 | next_frame_types_.resize( |
| 730 | std::max(static_cast<int>(codec.numberOfSimulcastStreams), 1), |
| 731 | kVideoFrameKey); |
| 732 | RTC_LOG(LS_VERBOSE) << " max bitrate " << codec.maxBitrate |
| 733 | << " start bitrate " << codec.startBitrate |
| 734 | << " max frame rate " << codec.maxFramerate |
| 735 | << " max payload size " << max_data_payload_length_; |
| 736 | } else { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 737 | RTC_LOG(LS_ERROR) << "Failed to configure encoder."; |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 738 | rate_allocator_ = nullptr; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 739 | } |
Peter Boström | 905f8e7 | 2016-03-02 16:59:56 +0100 | [diff] [blame] | 740 | |
Mirta Dvornicic | ccc1b57 | 2019-01-15 12:42:18 +0100 | [diff] [blame] | 741 | if (pending_encoder_creation_) { |
| 742 | overuse_detector_->StopCheckForOveruse(); |
| 743 | overuse_detector_->StartCheckForOveruse( |
Sebastian Jansson | cda86dd | 2019-03-11 17:26:36 +0100 | [diff] [blame^] | 744 | &encoder_queue_, |
Mirta Dvornicic | ccc1b57 | 2019-01-15 12:42:18 +0100 | [diff] [blame] | 745 | GetCpuOveruseOptions( |
| 746 | settings_, encoder_->GetEncoderInfo().is_hardware_accelerated), |
| 747 | this); |
| 748 | pending_encoder_creation_ = false; |
| 749 | } |
| 750 | |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 751 | int num_layers; |
| 752 | if (codec.codecType == kVideoCodecVP8) { |
| 753 | num_layers = codec.VP8()->numberOfTemporalLayers; |
| 754 | } else if (codec.codecType == kVideoCodecVP9) { |
| 755 | num_layers = codec.VP9()->numberOfTemporalLayers; |
Johnny Lee | 1a1c52b | 2019-02-08 14:25:40 -0500 | [diff] [blame] | 756 | } else if (codec.codecType == kVideoCodecH264) { |
| 757 | num_layers = codec.H264()->numberOfTemporalLayers; |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 758 | } else if (codec.codecType == kVideoCodecGeneric && |
| 759 | codec.numberOfSimulcastStreams > 0) { |
| 760 | // This is mainly for unit testing, disabling frame dropping. |
| 761 | // TODO(sprang): Add a better way to disable frame dropping. |
| 762 | num_layers = codec.simulcastStream[0].numberOfTemporalLayers; |
| 763 | } else { |
| 764 | num_layers = 1; |
| 765 | } |
| 766 | |
| 767 | frame_dropper_.Reset(); |
| 768 | frame_dropper_.SetRates(codec.startBitrate, max_framerate_); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 769 | // Force-disable frame dropper if either: |
| 770 | // * We have screensharing with layers. |
| 771 | // * "WebRTC-FrameDropper" field trial is "Disabled". |
| 772 | force_disable_frame_dropper_ = |
| 773 | field_trial::IsDisabled(kFrameDropperFieldTrial) || |
| 774 | (num_layers > 1 && codec.mode == VideoCodecMode::kScreensharing); |
| 775 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 776 | VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo(); |
| 777 | if (rate_control_settings_.UseEncoderBitrateAdjuster()) { |
| 778 | bitrate_adjuster_ = absl::make_unique<EncoderBitrateAdjuster>(codec); |
| 779 | bitrate_adjuster_->OnEncoderInfo(info); |
| 780 | } |
| 781 | |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 782 | if (rate_allocator_ && last_observed_bitrate_bps_ > 0) { |
| 783 | // We have a new rate allocator instance and already configured target |
| 784 | // bitrate. Update the rate allocation and notify observsers. |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 785 | const uint32_t framerate_fps = GetInputFramerateFps(); |
| 786 | SetEncoderRates(GetBitrateAllocationAndNotifyObserver( |
| 787 | last_observed_bitrate_bps_, framerate_fps), |
| 788 | framerate_fps); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 789 | } |
ilnik | 35b7de4 | 2017-03-15 04:24:21 -0700 | [diff] [blame] | 790 | |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 791 | encoder_stats_observer_->OnEncoderReconfigured(encoder_config_, streams); |
Per | 512ecb3 | 2016-09-23 15:52:06 +0200 | [diff] [blame] | 792 | |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 793 | pending_encoder_reconfiguration_ = false; |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 794 | |
Per | a48ddb7 | 2016-09-29 11:48:50 +0200 | [diff] [blame] | 795 | sink_->OnEncoderConfigurationChanged( |
Rasmus Brandt | c402dbe | 2019-02-04 11:09:46 +0100 | [diff] [blame] | 796 | std::move(streams), encoder_config_.content_type, |
| 797 | encoder_config_.min_transmit_bitrate_bps); |
kthelgason | 876222f | 2016-11-29 01:44:11 -0800 | [diff] [blame] | 798 | |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 799 | // Get the current target framerate, ie the maximum framerate as specified by |
| 800 | // the current codec configuration, or any limit imposed by cpu adaption in |
| 801 | // maintain-resolution or balanced mode. This is used to make sure overuse |
| 802 | // detection doesn't needlessly trigger in low and/or variable framerate |
| 803 | // scenarios. |
| 804 | int target_framerate = std::min( |
| 805 | max_framerate_, source_proxy_->GetActiveSinkWants().max_framerate_fps); |
| 806 | overuse_detector_->OnTargetFramerateUpdated(target_framerate); |
Niels Möller | 2d06118 | 2018-04-24 09:13:08 +0200 | [diff] [blame] | 807 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 808 | ConfigureQualityScaler(info); |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 809 | } |
| 810 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 811 | void VideoStreamEncoder::ConfigureQualityScaler( |
| 812 | const VideoEncoder::EncoderInfo& encoder_info) { |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 813 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 814 | const auto scaling_settings = encoder_info.scaling_settings; |
asapersson | 36e9eb4 | 2017-03-31 05:29:12 -0700 | [diff] [blame] | 815 | const bool quality_scaling_allowed = |
asapersson | 91914e2 | 2017-06-01 00:34:08 -0700 | [diff] [blame] | 816 | IsResolutionScalingEnabled(degradation_preference_) && |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 817 | scaling_settings.thresholds; |
kthelgason | 3af6cc0 | 2017-03-22 00:25:28 -0700 | [diff] [blame] | 818 | |
asapersson | 36e9eb4 | 2017-03-31 05:29:12 -0700 | [diff] [blame] | 819 | if (quality_scaling_allowed) { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 820 | if (quality_scaler_.get() == nullptr) { |
| 821 | // Quality scaler has not already been configured. |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 822 | |
Åsa Persson | a945aee | 2018-04-24 16:53:25 +0200 | [diff] [blame] | 823 | // Use experimental thresholds if available. |
Danil Chapovalov | b9b146c | 2018-06-15 12:28:07 +0200 | [diff] [blame] | 824 | absl::optional<VideoEncoder::QpThresholds> experimental_thresholds; |
Åsa Persson | a945aee | 2018-04-24 16:53:25 +0200 | [diff] [blame] | 825 | if (quality_scaling_experiment_enabled_) { |
| 826 | experimental_thresholds = QualityScalingExperiment::GetQpThresholds( |
| 827 | encoder_config_.codec_type); |
| 828 | } |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 829 | // Since the interface is non-public, absl::make_unique can't do this |
| 830 | // upcast. |
Niels Möller | 225c787 | 2018-02-22 15:03:53 +0100 | [diff] [blame] | 831 | AdaptationObserverInterface* observer = this; |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 832 | quality_scaler_ = absl::make_unique<QualityScaler>( |
Sebastian Jansson | cda86dd | 2019-03-11 17:26:36 +0100 | [diff] [blame^] | 833 | &encoder_queue_, observer, |
| 834 | experimental_thresholds ? *experimental_thresholds |
| 835 | : *(scaling_settings.thresholds)); |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 836 | has_seen_first_significant_bwe_change_ = false; |
| 837 | initial_framedrop_ = 0; |
kthelgason | 876222f | 2016-11-29 01:44:11 -0800 | [diff] [blame] | 838 | } |
| 839 | } else { |
| 840 | quality_scaler_.reset(nullptr); |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 841 | initial_framedrop_ = kMaxInitialFramedrop; |
kthelgason | 876222f | 2016-11-29 01:44:11 -0800 | [diff] [blame] | 842 | } |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 843 | |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 844 | encoder_stats_observer_->OnAdaptationChanged( |
| 845 | VideoStreamEncoderObserver::AdaptationReason::kNone, |
| 846 | GetActiveCounts(kCpu), GetActiveCounts(kQuality)); |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 847 | } |
| 848 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 849 | void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 850 | RTC_DCHECK_RUNS_SERIALIZED(&incoming_frame_race_checker_); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 851 | VideoFrame incoming_frame = video_frame; |
| 852 | |
| 853 | // Local time in webrtc time base. |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 854 | int64_t current_time_us = clock_->TimeInMicroseconds(); |
| 855 | int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec; |
| 856 | // In some cases, e.g., when the frame from decoder is fed to encoder, |
| 857 | // the timestamp may be set to the future. As the encoding pipeline assumes |
| 858 | // capture time to be less than present time, we should reset the capture |
| 859 | // timestamps here. Otherwise there may be issues with RTP send stream. |
| 860 | if (incoming_frame.timestamp_us() > current_time_us) |
| 861 | incoming_frame.set_timestamp_us(current_time_us); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 862 | |
| 863 | // Capture time may come from clock with an offset and drift from clock_. |
| 864 | int64_t capture_ntp_time_ms; |
nisse | 891419f | 2017-01-12 10:02:22 -0800 | [diff] [blame] | 865 | if (video_frame.ntp_time_ms() > 0) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 866 | capture_ntp_time_ms = video_frame.ntp_time_ms(); |
| 867 | } else if (video_frame.render_time_ms() != 0) { |
| 868 | capture_ntp_time_ms = video_frame.render_time_ms() + delta_ntp_internal_ms_; |
| 869 | } else { |
nisse | 1c0dea8 | 2017-01-30 02:43:18 -0800 | [diff] [blame] | 870 | capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_; |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 871 | } |
| 872 | incoming_frame.set_ntp_time_ms(capture_ntp_time_ms); |
| 873 | |
| 874 | // Convert NTP time, in ms, to RTP timestamp. |
| 875 | const int kMsToRtpTimestamp = 90; |
| 876 | incoming_frame.set_timestamp( |
| 877 | kMsToRtpTimestamp * static_cast<uint32_t>(incoming_frame.ntp_time_ms())); |
| 878 | |
| 879 | if (incoming_frame.ntp_time_ms() <= last_captured_timestamp_) { |
| 880 | // We don't allow the same capture time for two frames, drop this one. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 881 | RTC_LOG(LS_WARNING) << "Same/old NTP timestamp (" |
| 882 | << incoming_frame.ntp_time_ms() |
| 883 | << " <= " << last_captured_timestamp_ |
| 884 | << ") for incoming frame. Dropping."; |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 885 | encoder_queue_.PostTask([this, incoming_frame]() { |
| 886 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 887 | accumulated_update_rect_.Union(incoming_frame.update_rect()); |
| 888 | }); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 889 | return; |
| 890 | } |
| 891 | |
asapersson | 6ffb67d | 2016-09-12 00:10:45 -0700 | [diff] [blame] | 892 | bool log_stats = false; |
nisse | 1c0dea8 | 2017-01-30 02:43:18 -0800 | [diff] [blame] | 893 | if (current_time_ms - last_frame_log_ms_ > kFrameLogIntervalMs) { |
| 894 | last_frame_log_ms_ = current_time_ms; |
asapersson | 6ffb67d | 2016-09-12 00:10:45 -0700 | [diff] [blame] | 895 | log_stats = true; |
| 896 | } |
| 897 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 898 | last_captured_timestamp_ = incoming_frame.ntp_time_ms(); |
Sebastian Jansson | 3ab5c40 | 2018-04-05 12:30:50 +0200 | [diff] [blame] | 899 | |
| 900 | int64_t post_time_us = rtc::TimeMicros(); |
| 901 | ++posted_frames_waiting_for_encode_; |
| 902 | |
| 903 | encoder_queue_.PostTask( |
| 904 | [this, incoming_frame, post_time_us, log_stats]() { |
| 905 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 906 | encoder_stats_observer_->OnIncomingFrame(incoming_frame.width(), |
| 907 | incoming_frame.height()); |
Sebastian Jansson | 3ab5c40 | 2018-04-05 12:30:50 +0200 | [diff] [blame] | 908 | ++captured_frame_count_; |
| 909 | const int posted_frames_waiting_for_encode = |
| 910 | posted_frames_waiting_for_encode_.fetch_sub(1); |
| 911 | RTC_DCHECK_GT(posted_frames_waiting_for_encode, 0); |
| 912 | if (posted_frames_waiting_for_encode == 1) { |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 913 | MaybeEncodeVideoFrame(incoming_frame, post_time_us); |
Sebastian Jansson | 3ab5c40 | 2018-04-05 12:30:50 +0200 | [diff] [blame] | 914 | } else { |
| 915 | // There is a newer frame in flight. Do not encode this frame. |
| 916 | RTC_LOG(LS_VERBOSE) |
| 917 | << "Incoming frame dropped due to that the encoder is blocked."; |
| 918 | ++dropped_frame_count_; |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 919 | encoder_stats_observer_->OnFrameDropped( |
| 920 | VideoStreamEncoderObserver::DropReason::kEncoderQueue); |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 921 | accumulated_update_rect_.Union(incoming_frame.update_rect()); |
Sebastian Jansson | 3ab5c40 | 2018-04-05 12:30:50 +0200 | [diff] [blame] | 922 | } |
| 923 | if (log_stats) { |
| 924 | RTC_LOG(LS_INFO) << "Number of frames: captured " |
| 925 | << captured_frame_count_ |
| 926 | << ", dropped (due to encoder blocked) " |
| 927 | << dropped_frame_count_ << ", interval_ms " |
| 928 | << kFrameLogIntervalMs; |
| 929 | captured_frame_count_ = 0; |
| 930 | dropped_frame_count_ = 0; |
| 931 | } |
| 932 | }); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 933 | } |
| 934 | |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 935 | void VideoStreamEncoder::OnDiscardedFrame() { |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 936 | encoder_stats_observer_->OnFrameDropped( |
| 937 | VideoStreamEncoderObserver::DropReason::kSource); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 938 | } |
| 939 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 940 | bool VideoStreamEncoder::EncoderPaused() const { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 941 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
pwestin@webrtc.org | 91563e4 | 2013-04-25 22:20:08 +0000 | [diff] [blame] | 942 | // Pause video if paused by caller or as long as the network is down or the |
| 943 | // pacer queue has grown too large in buffered mode. |
perkj | 57c21f9 | 2016-06-17 07:27:16 -0700 | [diff] [blame] | 944 | // If the pacer queue has grown too large or the network is down, |
perkj | fea9309 | 2016-05-14 00:58:48 -0700 | [diff] [blame] | 945 | // last_observed_bitrate_bps_ will be 0. |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 946 | return last_observed_bitrate_bps_ == 0; |
stefan@webrtc.org | bfacda6 | 2013-03-27 16:36:01 +0000 | [diff] [blame] | 947 | } |
| 948 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 949 | void VideoStreamEncoder::TraceFrameDropStart() { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 950 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 +0000 | [diff] [blame] | 951 | // Start trace event only on the first frame after encoder is paused. |
| 952 | if (!encoder_paused_and_dropped_frame_) { |
| 953 | TRACE_EVENT_ASYNC_BEGIN0("webrtc", "EncoderPaused", this); |
| 954 | } |
| 955 | encoder_paused_and_dropped_frame_ = true; |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 +0000 | [diff] [blame] | 956 | } |
| 957 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 958 | void VideoStreamEncoder::TraceFrameDropEnd() { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 959 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
sprang@webrtc.org | dcebf2d | 2014-11-04 16:27:16 +0000 | [diff] [blame] | 960 | // End trace event on first frame after encoder resumes, if frame was dropped. |
| 961 | if (encoder_paused_and_dropped_frame_) { |
| 962 | TRACE_EVENT_ASYNC_END0("webrtc", "EncoderPaused", this); |
| 963 | } |
| 964 | encoder_paused_and_dropped_frame_ = false; |
| 965 | } |
| 966 | |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 967 | VideoBitrateAllocation |
| 968 | VideoStreamEncoder::GetBitrateAllocationAndNotifyObserver( |
| 969 | const uint32_t target_bitrate_bps, |
| 970 | uint32_t framerate_fps) { |
| 971 | // Only call allocators if bitrate > 0 (ie, not suspended), otherwise they |
| 972 | // might cap the bitrate to the min bitrate configured. |
| 973 | VideoBitrateAllocation bitrate_allocation; |
| 974 | if (rate_allocator_ && target_bitrate_bps > 0) { |
| 975 | bitrate_allocation = |
| 976 | rate_allocator_->GetAllocation(target_bitrate_bps, framerate_fps); |
| 977 | } |
| 978 | |
| 979 | if (bitrate_observer_ && bitrate_allocation.get_sum_bps() > 0) { |
| 980 | bitrate_observer_->OnBitrateAllocationUpdated(bitrate_allocation); |
| 981 | } |
| 982 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 983 | if (bitrate_adjuster_) { |
Erik Språng | 0e1a1f9 | 2019-02-18 18:45:13 +0100 | [diff] [blame] | 984 | VideoBitrateAllocation adjusted_allocation = |
| 985 | bitrate_adjuster_->AdjustRateAllocation(bitrate_allocation, |
| 986 | framerate_fps); |
| 987 | RTC_LOG(LS_VERBOSE) << "Adjusting allocation, fps = " << framerate_fps |
| 988 | << ", from " << bitrate_allocation.ToString() << ", to " |
| 989 | << adjusted_allocation.ToString(); |
| 990 | return adjusted_allocation; |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 991 | } |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 992 | return bitrate_allocation; |
| 993 | } |
| 994 | |
| 995 | uint32_t VideoStreamEncoder::GetInputFramerateFps() { |
| 996 | const uint32_t default_fps = max_framerate_ != -1 ? max_framerate_ : 30; |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 997 | absl::optional<uint32_t> input_fps = |
| 998 | input_framerate_.Rate(clock_->TimeInMilliseconds()); |
| 999 | if (!input_fps || *input_fps == 0) { |
| 1000 | return default_fps; |
| 1001 | } |
| 1002 | return *input_fps; |
| 1003 | } |
| 1004 | |
| 1005 | void VideoStreamEncoder::SetEncoderRates( |
| 1006 | const VideoBitrateAllocation& bitrate_allocation, |
| 1007 | uint32_t framerate_fps) { |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1008 | if (!encoder_) { |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1009 | return; |
| 1010 | } |
| 1011 | |
| 1012 | // |bitrate_allocation| is 0 it means that the network is down or the send |
| 1013 | // pacer is full. We currently only report this if the encoder has an internal |
| 1014 | // source. If the encoder does not have an internal source, higher levels |
| 1015 | // are expected to not call AddVideoFrame. We do this since its unclear |
| 1016 | // how current encoder implementations behave when given a zero target |
| 1017 | // bitrate. |
| 1018 | // TODO(perkj): Make sure all known encoder implementations handle zero |
| 1019 | // target bitrate and remove this check. |
| 1020 | if (!HasInternalSource() && bitrate_allocation.get_sum_bps() == 0) { |
| 1021 | return; |
| 1022 | } |
| 1023 | |
| 1024 | RTC_DCHECK_GT(framerate_fps, 0); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1025 | if (bitrate_allocation != last_bitrate_allocation_ || |
| 1026 | framerate_fps != last_framerate_fps_) { |
| 1027 | int res = encoder_->SetRateAllocation(bitrate_allocation, framerate_fps); |
| 1028 | if (res != 0) { |
| 1029 | RTC_LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = " |
| 1030 | << bitrate_allocation.get_sum_bps() |
| 1031 | << ", framerate = " << framerate_fps << "): " << res; |
| 1032 | } |
| 1033 | |
| 1034 | frame_encoder_timer_.OnSetRates(bitrate_allocation, framerate_fps); |
| 1035 | } |
| 1036 | |
| 1037 | last_bitrate_allocation_ = bitrate_allocation; |
| 1038 | last_framerate_fps_ = framerate_fps; |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1039 | } |
| 1040 | |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1041 | void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, |
| 1042 | int64_t time_when_posted_us) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1043 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
kthelgason | 876222f | 2016-11-29 01:44:11 -0800 | [diff] [blame] | 1044 | |
Per | 21d45d2 | 2016-10-30 21:37:57 +0100 | [diff] [blame] | 1045 | if (!last_frame_info_ || video_frame.width() != last_frame_info_->width || |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 1046 | video_frame.height() != last_frame_info_->height || |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 1047 | video_frame.is_texture() != last_frame_info_->is_texture) { |
| 1048 | pending_encoder_reconfiguration_ = true; |
Oskar Sundbom | 8e07c13 | 2018-01-08 16:45:42 +0100 | [diff] [blame] | 1049 | last_frame_info_ = VideoFrameInfo(video_frame.width(), video_frame.height(), |
| 1050 | video_frame.is_texture()); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1051 | RTC_LOG(LS_INFO) << "Video frame parameters changed: dimensions=" |
| 1052 | << last_frame_info_->width << "x" |
| 1053 | << last_frame_info_->height |
| 1054 | << ", texture=" << last_frame_info_->is_texture << "."; |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1055 | // Force full frame update, since resolution has changed. |
| 1056 | accumulated_update_rect_ = |
| 1057 | VideoFrame::UpdateRect{0, 0, video_frame.width(), video_frame.height()}; |
perkj | fa10b55 | 2016-10-02 23:45:26 -0700 | [diff] [blame] | 1058 | } |
| 1059 | |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 1060 | // We have to create then encoder before the frame drop logic, |
| 1061 | // because the latter depends on encoder_->GetScalingSettings. |
| 1062 | // According to the testcase |
| 1063 | // InitialFrameDropOffWhenEncoderDisabledScaling, the return value |
| 1064 | // from GetScalingSettings should enable or disable the frame drop. |
| 1065 | |
Erik Språng | a8d48ab | 2019-02-08 14:17:40 +0100 | [diff] [blame] | 1066 | // Update input frame rate before we start using it. If we update it after |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1067 | // any potential frame drop we are going to artificially increase frame sizes. |
| 1068 | // Poll the rate before updating, otherwise we risk the rate being estimated |
| 1069 | // a little too high at the start of the call when then window is small. |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1070 | uint32_t framerate_fps = GetInputFramerateFps(); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1071 | input_framerate_.Update(1u, clock_->TimeInMilliseconds()); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1072 | |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 1073 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 1074 | if (pending_encoder_reconfiguration_) { |
| 1075 | ReconfigureEncoder(); |
| 1076 | last_parameters_update_ms_.emplace(now_ms); |
| 1077 | } else if (!last_parameters_update_ms_ || |
| 1078 | now_ms - *last_parameters_update_ms_ >= |
| 1079 | vcm::VCMProcessTimer::kDefaultProcessIntervalMs) { |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1080 | SetEncoderRates(GetBitrateAllocationAndNotifyObserver( |
| 1081 | last_observed_bitrate_bps_, framerate_fps), |
| 1082 | framerate_fps); |
Niels Möller | 4db138e | 2018-04-19 09:04:13 +0200 | [diff] [blame] | 1083 | last_parameters_update_ms_.emplace(now_ms); |
| 1084 | } |
| 1085 | |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1086 | // Because pending frame will be dropped in any case, we need to |
| 1087 | // remember its updated region. |
| 1088 | if (pending_frame_) { |
| 1089 | encoder_stats_observer_->OnFrameDropped( |
| 1090 | VideoStreamEncoderObserver::DropReason::kEncoderQueue); |
| 1091 | accumulated_update_rect_.Union(pending_frame_->update_rect()); |
| 1092 | } |
| 1093 | |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1094 | if (DropDueToSize(video_frame.size())) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1095 | RTC_LOG(LS_INFO) << "Dropping frame. Too large for target bitrate."; |
Åsa Persson | 875841d | 2018-01-08 08:49:53 +0100 | [diff] [blame] | 1096 | int count = GetConstAdaptCounter().ResolutionCount(kQuality); |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 1097 | AdaptDown(kQuality); |
Åsa Persson | 875841d | 2018-01-08 08:49:53 +0100 | [diff] [blame] | 1098 | if (GetConstAdaptCounter().ResolutionCount(kQuality) > count) { |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1099 | encoder_stats_observer_->OnInitialQualityResolutionAdaptDown(); |
Åsa Persson | 875841d | 2018-01-08 08:49:53 +0100 | [diff] [blame] | 1100 | } |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 1101 | ++initial_framedrop_; |
Sebastian Jansson | 0d70e37 | 2018-04-17 13:57:13 +0200 | [diff] [blame] | 1102 | // Storing references to a native buffer risks blocking frame capture. |
| 1103 | if (video_frame.video_frame_buffer()->type() != |
| 1104 | VideoFrameBuffer::Type::kNative) { |
| 1105 | pending_frame_ = video_frame; |
| 1106 | pending_frame_post_time_us_ = time_when_posted_us; |
| 1107 | } else { |
| 1108 | // Ensure that any previously stored frame is dropped. |
| 1109 | pending_frame_.reset(); |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1110 | accumulated_update_rect_.Union(video_frame.update_rect()); |
Sebastian Jansson | 0d70e37 | 2018-04-17 13:57:13 +0200 | [diff] [blame] | 1111 | } |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 1112 | return; |
| 1113 | } |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 1114 | initial_framedrop_ = kMaxInitialFramedrop; |
kthelgason | 2bc6864 | 2017-02-07 07:02:22 -0800 | [diff] [blame] | 1115 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1116 | if (EncoderPaused()) { |
Sebastian Jansson | 0d70e37 | 2018-04-17 13:57:13 +0200 | [diff] [blame] | 1117 | // Storing references to a native buffer risks blocking frame capture. |
| 1118 | if (video_frame.video_frame_buffer()->type() != |
| 1119 | VideoFrameBuffer::Type::kNative) { |
| 1120 | if (pending_frame_) |
| 1121 | TraceFrameDropStart(); |
| 1122 | pending_frame_ = video_frame; |
| 1123 | pending_frame_post_time_us_ = time_when_posted_us; |
| 1124 | } else { |
| 1125 | // Ensure that any previously stored frame is dropped. |
| 1126 | pending_frame_.reset(); |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1127 | TraceFrameDropStart(); |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1128 | accumulated_update_rect_.Union(video_frame.update_rect()); |
Sebastian Jansson | 0d70e37 | 2018-04-17 13:57:13 +0200 | [diff] [blame] | 1129 | } |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1130 | return; |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 1131 | } |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1132 | |
| 1133 | pending_frame_.reset(); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1134 | |
| 1135 | frame_dropper_.Leak(framerate_fps); |
| 1136 | // Frame dropping is enabled iff frame dropping is not force-disabled, and |
| 1137 | // rate controller is not trusted. |
| 1138 | const bool frame_dropping_enabled = |
| 1139 | !force_disable_frame_dropper_ && |
| 1140 | !encoder_info_.has_trusted_rate_controller; |
| 1141 | frame_dropper_.Enable(frame_dropping_enabled); |
| 1142 | if (frame_dropping_enabled && frame_dropper_.DropFrame()) { |
| 1143 | RTC_LOG(LS_VERBOSE) << "Drop Frame: " |
| 1144 | << "target bitrate " << last_observed_bitrate_bps_ |
| 1145 | << ", input frame rate " << framerate_fps; |
| 1146 | OnDroppedFrame( |
| 1147 | EncodedImageCallback::DropReason::kDroppedByMediaOptimizations); |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1148 | accumulated_update_rect_.Union(video_frame.update_rect()); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1149 | return; |
| 1150 | } |
| 1151 | |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1152 | EncodeVideoFrame(video_frame, time_when_posted_us); |
| 1153 | } |
| 1154 | |
| 1155 | void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame, |
| 1156 | int64_t time_when_posted_us) { |
| 1157 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1158 | TraceFrameDropEnd(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1159 | |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 1160 | VideoFrame out_frame(video_frame); |
| 1161 | // Crop frame if needed. |
| 1162 | if (crop_width_ > 0 || crop_height_ > 0) { |
| 1163 | int cropped_width = video_frame.width() - crop_width_; |
| 1164 | int cropped_height = video_frame.height() - crop_height_; |
| 1165 | rtc::scoped_refptr<I420Buffer> cropped_buffer = |
| 1166 | I420Buffer::Create(cropped_width, cropped_height); |
| 1167 | // TODO(ilnik): Remove scaling if cropping is too big, as it should never |
| 1168 | // happen after SinkWants signaled correctly from ReconfigureEncoder. |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1169 | VideoFrame::UpdateRect update_rect = video_frame.update_rect(); |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 1170 | if (crop_width_ < 4 && crop_height_ < 4) { |
| 1171 | cropped_buffer->CropAndScaleFrom( |
| 1172 | *video_frame.video_frame_buffer()->ToI420(), crop_width_ / 2, |
| 1173 | crop_height_ / 2, cropped_width, cropped_height); |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1174 | update_rect.offset_x -= crop_width_ / 2; |
| 1175 | update_rect.offset_y -= crop_height_ / 2; |
| 1176 | update_rect.Intersect( |
| 1177 | VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height}); |
| 1178 | |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 1179 | } else { |
| 1180 | cropped_buffer->ScaleFrom( |
| 1181 | *video_frame.video_frame_buffer()->ToI420().get()); |
Ilya Nikolaevskiy | 1c90cab | 2019-03-07 15:30:58 +0100 | [diff] [blame] | 1182 | if (!update_rect.IsEmpty()) { |
| 1183 | // Since we can't reason about pixels after scaling, we invalidate whole |
| 1184 | // picture, if anything changed. |
| 1185 | update_rect = |
| 1186 | VideoFrame::UpdateRect{0, 0, cropped_width, cropped_height}; |
| 1187 | } |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 1188 | } |
Artem Titov | 1ebfb6a | 2019-01-03 23:49:37 +0100 | [diff] [blame] | 1189 | out_frame = VideoFrame::Builder() |
| 1190 | .set_video_frame_buffer(cropped_buffer) |
| 1191 | .set_timestamp_rtp(video_frame.timestamp()) |
| 1192 | .set_timestamp_ms(video_frame.render_time_ms()) |
| 1193 | .set_rotation(video_frame.rotation()) |
| 1194 | .set_id(video_frame.id()) |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1195 | .set_update_rect(update_rect) |
Artem Titov | 1ebfb6a | 2019-01-03 23:49:37 +0100 | [diff] [blame] | 1196 | .build(); |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 1197 | out_frame.set_ntp_time_ms(video_frame.ntp_time_ms()); |
Ilya Nikolaevskiy | 71aee3a | 2019-02-18 13:01:26 +0100 | [diff] [blame] | 1198 | // Since accumulated_update_rect_ is constructed before cropping, |
| 1199 | // we can't trust it. If any changes were pending, we invalidate whole |
| 1200 | // frame here. |
| 1201 | if (!accumulated_update_rect_.IsEmpty()) { |
| 1202 | accumulated_update_rect_ = |
| 1203 | VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()}; |
| 1204 | } |
| 1205 | } |
| 1206 | |
| 1207 | if (!accumulated_update_rect_.IsEmpty()) { |
| 1208 | accumulated_update_rect_.Union(out_frame.update_rect()); |
| 1209 | accumulated_update_rect_.Intersect( |
| 1210 | VideoFrame::UpdateRect{0, 0, out_frame.width(), out_frame.height()}); |
| 1211 | out_frame.set_update_rect(accumulated_update_rect_); |
| 1212 | accumulated_update_rect_.MakeEmptyUpdate(); |
ilnik | 6b826ef | 2017-06-16 06:53:48 -0700 | [diff] [blame] | 1213 | } |
| 1214 | |
Magnus Jedvert | 26679d6 | 2015-04-07 14:07:41 +0200 | [diff] [blame] | 1215 | TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", video_frame.render_time_ms(), |
hclam@chromium.org | 1a7b9b9 | 2013-07-08 21:31:18 +0000 | [diff] [blame] | 1216 | "Encode"); |
pbos@webrtc.org | fe1ef93 | 2013-10-21 10:34:43 +0000 | [diff] [blame] | 1217 | |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 1218 | overuse_detector_->FrameCaptured(out_frame, time_when_posted_us); |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 1219 | |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 1220 | // Encoder metadata needs to be updated before encode complete callback. |
| 1221 | VideoEncoder::EncoderInfo info = encoder_->GetEncoderInfo(); |
| 1222 | if (info.implementation_name != encoder_info_.implementation_name) { |
| 1223 | encoder_stats_observer_->OnEncoderImplementationChanged( |
| 1224 | info.implementation_name); |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1225 | if (bitrate_adjuster_) { |
| 1226 | // Encoder implementation changed, reset overshoot detector states. |
| 1227 | bitrate_adjuster_->Reset(); |
| 1228 | } |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 1229 | } |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1230 | |
| 1231 | if (bitrate_adjuster_) { |
| 1232 | for (size_t si = 0; si < kMaxSpatialLayers; ++si) { |
| 1233 | if (info.fps_allocation[si] != encoder_info_.fps_allocation[si]) { |
| 1234 | bitrate_adjuster_->OnEncoderInfo(info); |
| 1235 | break; |
| 1236 | } |
| 1237 | } |
| 1238 | } |
| 1239 | |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 1240 | encoder_info_ = info; |
Erik Språng | b7cb7b5 | 2019-02-26 15:52:33 +0100 | [diff] [blame] | 1241 | RTC_DCHECK_EQ(send_codec_.width, out_frame.width()); |
| 1242 | RTC_DCHECK_EQ(send_codec_.height, out_frame.height()); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1243 | const VideoFrameBuffer::Type buffer_type = |
| 1244 | out_frame.video_frame_buffer()->type(); |
| 1245 | const bool is_buffer_type_supported = |
| 1246 | buffer_type == VideoFrameBuffer::Type::kI420 || |
| 1247 | (buffer_type == VideoFrameBuffer::Type::kNative && |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1248 | info.supports_native_handle); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1249 | |
| 1250 | if (!is_buffer_type_supported) { |
| 1251 | // This module only supports software encoding. |
| 1252 | rtc::scoped_refptr<I420BufferInterface> converted_buffer( |
| 1253 | out_frame.video_frame_buffer()->ToI420()); |
| 1254 | |
| 1255 | if (!converted_buffer) { |
| 1256 | RTC_LOG(LS_ERROR) << "Frame conversion failed, dropping frame."; |
| 1257 | return; |
| 1258 | } |
| 1259 | |
Ilya Nikolaevskiy | 1c90cab | 2019-03-07 15:30:58 +0100 | [diff] [blame] | 1260 | // UpdatedRect is reset to full update if it's not empty, because buffer was |
| 1261 | // converted, therefore we can't guarantee that pixels outside of UpdateRect |
| 1262 | // didn't change comparing to the previous frame. |
| 1263 | VideoFrame::UpdateRect update_rect = |
| 1264 | out_frame.update_rect().IsEmpty() |
| 1265 | ? out_frame.update_rect() |
| 1266 | : VideoFrame::UpdateRect{0, 0, out_frame.width(), |
| 1267 | out_frame.height()}; |
| 1268 | |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1269 | out_frame = VideoFrame::Builder() |
| 1270 | .set_video_frame_buffer(converted_buffer) |
| 1271 | .set_timestamp_rtp(out_frame.timestamp()) |
| 1272 | .set_timestamp_ms(out_frame.render_time_ms()) |
| 1273 | .set_rotation(out_frame.rotation()) |
| 1274 | .set_id(out_frame.id()) |
Ilya Nikolaevskiy | 1c90cab | 2019-03-07 15:30:58 +0100 | [diff] [blame] | 1275 | .set_update_rect(update_rect) |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1276 | .build(); |
| 1277 | } |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1278 | |
| 1279 | TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp", |
| 1280 | out_frame.timestamp()); |
| 1281 | |
| 1282 | frame_encoder_timer_.OnEncodeStarted(out_frame.timestamp(), |
| 1283 | out_frame.render_time_ms()); |
| 1284 | |
Niels Möller | c8d2e73 | 2019-03-06 12:00:33 +0100 | [diff] [blame] | 1285 | const int32_t encode_status = encoder_->Encode(out_frame, &next_frame_types_); |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1286 | |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1287 | if (encode_status < 0) { |
| 1288 | RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: " |
| 1289 | << encode_status; |
| 1290 | return; |
| 1291 | } |
| 1292 | |
| 1293 | for (auto& it : next_frame_types_) { |
| 1294 | it = kVideoFrameDelta; |
| 1295 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1296 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1297 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1298 | void VideoStreamEncoder::SendKeyFrame() { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1299 | if (!encoder_queue_.IsCurrent()) { |
| 1300 | encoder_queue_.PostTask([this] { SendKeyFrame(); }); |
| 1301 | return; |
| 1302 | } |
| 1303 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
Niels Möller | 1c9aa1e | 2018-02-16 10:27:23 +0100 | [diff] [blame] | 1304 | TRACE_EVENT0("webrtc", "OnKeyFrameRequest"); |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1305 | RTC_DCHECK(!next_frame_types_.empty()); |
| 1306 | next_frame_types_[0] = kVideoFrameKey; |
| 1307 | if (HasInternalSource()) { |
| 1308 | // Try to request the frame if we have an external encoder with |
| 1309 | // internal source since AddVideoFrame never will be called. |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1310 | |
| 1311 | // TODO(nisse): Used only with internal source. Delete as soon as |
| 1312 | // that feature is removed. The only implementation I've been able |
| 1313 | // to find ignores what's in the frame. With one exception: It seems |
| 1314 | // a few test cases, e.g., |
| 1315 | // VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set |
| 1316 | // internal_source to true and use FakeEncoder. And the latter will |
| 1317 | // happily encode this 1x1 frame and pass it on down the pipeline. |
| 1318 | if (encoder_->Encode(VideoFrame::Builder() |
| 1319 | .set_video_frame_buffer(I420Buffer::Create(1, 1)) |
| 1320 | .set_rotation(kVideoRotation_0) |
| 1321 | .set_timestamp_us(0) |
| 1322 | .build(), |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1323 | &next_frame_types_) == WEBRTC_VIDEO_CODEC_OK) { |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1324 | // Try to remove just-performed keyframe request, if stream still exists. |
| 1325 | next_frame_types_[0] = kVideoFrameDelta; |
| 1326 | } |
| 1327 | } |
stefan@webrtc.org | 07b45a5 | 2012-02-02 08:37:48 +0000 | [diff] [blame] | 1328 | } |
| 1329 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1330 | EncodedImageCallback::Result VideoStreamEncoder::OnEncodedImage( |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 1331 | const EncodedImage& encoded_image, |
| 1332 | const CodecSpecificInfo* codec_specific_info, |
| 1333 | const RTPFragmentationHeader* fragmentation) { |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1334 | const int64_t time_sent_us = rtc::TimeMicros(); |
| 1335 | |
| 1336 | TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded", |
| 1337 | "timestamp", encoded_image.Timestamp()); |
| 1338 | const size_t spatial_idx = encoded_image.SpatialIndex().value_or(0); |
| 1339 | EncodedImage image_copy(encoded_image); |
| 1340 | |
| 1341 | frame_encoder_timer_.FillTimingInfo( |
| 1342 | spatial_idx, &image_copy, time_sent_us / rtc::kNumMicrosecsPerMillisec); |
| 1343 | |
| 1344 | // Piggyback ALR experiment group id and simulcast id into the content type. |
| 1345 | const uint8_t experiment_id = |
| 1346 | experiment_groups_[videocontenttypehelpers::IsScreenshare( |
| 1347 | image_copy.content_type_)]; |
| 1348 | |
| 1349 | // TODO(ilnik): This will force content type extension to be present even |
| 1350 | // for realtime video. At the expense of miniscule overhead we will get |
| 1351 | // sliced receive statistics. |
| 1352 | RTC_CHECK(videocontenttypehelpers::SetExperimentId(&image_copy.content_type_, |
| 1353 | experiment_id)); |
| 1354 | // We count simulcast streams from 1 on the wire. That's why we set simulcast |
| 1355 | // id in content type to +1 of that is actual simulcast index. This is because |
| 1356 | // value 0 on the wire is reserved for 'no simulcast stream specified'. |
| 1357 | RTC_CHECK(videocontenttypehelpers::SetSimulcastId( |
| 1358 | &image_copy.content_type_, static_cast<uint8_t>(spatial_idx + 1))); |
| 1359 | |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1360 | // Encoded is called on whatever thread the real encoder implementation run |
| 1361 | // on. In the case of hardware encoders, there might be several encoders |
| 1362 | // running in parallel on different threads. |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1363 | encoder_stats_observer_->OnSendEncodedImage(image_copy, codec_specific_info); |
sprang | 3911c26 | 2016-04-15 01:24:14 -0700 | [diff] [blame] | 1364 | |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 1365 | EncodedImageCallback::Result result = |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1366 | sink_->OnEncodedImage(image_copy, codec_specific_info, fragmentation); |
perkj | bc75d97 | 2016-05-02 06:31:25 -0700 | [diff] [blame] | 1367 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1368 | // We are only interested in propagating the meta-data about the image, not |
| 1369 | // encoded data itself, to the post encode function. Since we cannot be sure |
| 1370 | // the pointer will still be valid when run on the task queue, set it to null. |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1371 | image_copy.set_buffer(nullptr, 0); |
Niels Möller | 83dbeac | 2017-12-14 16:39:44 +0100 | [diff] [blame] | 1372 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1373 | int temporal_index = 0; |
| 1374 | if (codec_specific_info) { |
| 1375 | if (codec_specific_info->codecType == kVideoCodecVP9) { |
| 1376 | temporal_index = codec_specific_info->codecSpecific.VP9.temporal_idx; |
| 1377 | } else if (codec_specific_info->codecType == kVideoCodecVP8) { |
| 1378 | temporal_index = codec_specific_info->codecSpecific.VP8.temporalIdx; |
| 1379 | } |
| 1380 | } |
| 1381 | if (temporal_index == kNoTemporalIdx) { |
| 1382 | temporal_index = 0; |
Niels Möller | 83dbeac | 2017-12-14 16:39:44 +0100 | [diff] [blame] | 1383 | } |
| 1384 | |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1385 | RunPostEncode(image_copy, time_sent_us, temporal_index); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1386 | |
| 1387 | if (result.error == Result::OK) { |
| 1388 | // In case of an internal encoder running on a separate thread, the |
| 1389 | // decision to drop a frame might be a frame late and signaled via |
| 1390 | // atomic flag. This is because we can't easily wait for the worker thread |
| 1391 | // without risking deadlocks, eg during shutdown when the worker thread |
| 1392 | // might be waiting for the internal encoder threads to stop. |
| 1393 | if (pending_frame_drops_.load() > 0) { |
| 1394 | int pending_drops = pending_frame_drops_.fetch_sub(1); |
| 1395 | RTC_DCHECK_GT(pending_drops, 0); |
| 1396 | result.drop_next_frame = true; |
| 1397 | } |
| 1398 | } |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 1399 | |
Sergey Ulanov | 525df3f | 2016-08-02 17:46:41 -0700 | [diff] [blame] | 1400 | return result; |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 1401 | } |
| 1402 | |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 1403 | void VideoStreamEncoder::OnDroppedFrame(DropReason reason) { |
| 1404 | switch (reason) { |
| 1405 | case DropReason::kDroppedByMediaOptimizations: |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1406 | encoder_stats_observer_->OnFrameDropped( |
| 1407 | VideoStreamEncoderObserver::DropReason::kMediaOptimization); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 1408 | encoder_queue_.PostTask([this] { |
| 1409 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 1410 | if (quality_scaler_) |
Åsa Persson | a945aee | 2018-04-24 16:53:25 +0200 | [diff] [blame] | 1411 | quality_scaler_->ReportDroppedFrameByMediaOpt(); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 1412 | }); |
| 1413 | break; |
| 1414 | case DropReason::kDroppedByEncoder: |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1415 | encoder_stats_observer_->OnFrameDropped( |
| 1416 | VideoStreamEncoderObserver::DropReason::kEncoder); |
Åsa Persson | a945aee | 2018-04-24 16:53:25 +0200 | [diff] [blame] | 1417 | encoder_queue_.PostTask([this] { |
| 1418 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 1419 | if (quality_scaler_) |
| 1420 | quality_scaler_->ReportDroppedFrameByEncoder(); |
| 1421 | }); |
Ilya Nikolaevskiy | d79314f | 2017-10-23 10:45:37 +0200 | [diff] [blame] | 1422 | break; |
| 1423 | } |
kthelgason | 876222f | 2016-11-29 01:44:11 -0800 | [diff] [blame] | 1424 | } |
| 1425 | |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1426 | void VideoStreamEncoder::OnBitrateUpdated(DataRate target_bitrate, |
| 1427 | DataRate target_headroom, |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1428 | uint8_t fraction_lost, |
| 1429 | int64_t round_trip_time_ms) { |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1430 | if (!encoder_queue_.IsCurrent()) { |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1431 | encoder_queue_.PostTask([this, target_bitrate, target_headroom, |
| 1432 | fraction_lost, round_trip_time_ms] { |
| 1433 | OnBitrateUpdated(target_bitrate, target_headroom, fraction_lost, |
| 1434 | round_trip_time_ms); |
| 1435 | }); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1436 | return; |
| 1437 | } |
| 1438 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
| 1439 | RTC_DCHECK(sink_) << "sink_ must be set before the encoder is active."; |
| 1440 | |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1441 | RTC_LOG(LS_VERBOSE) << "OnBitrateUpdated, bitrate " << target_bitrate.bps() |
| 1442 | << " headroom = " << target_headroom.bps() |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1443 | << " packet loss " << static_cast<int>(fraction_lost) |
| 1444 | << " rtt " << round_trip_time_ms; |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 1445 | // On significant changes to BWE at the start of the call, |
| 1446 | // enable frame drops to quickly react to jumps in available bandwidth. |
| 1447 | if (encoder_start_bitrate_bps_ != 0 && |
| 1448 | !has_seen_first_significant_bwe_change_ && quality_scaler_ && |
| 1449 | initial_framedrop_on_bwe_enabled_ && |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1450 | abs_diff(target_bitrate.bps(), encoder_start_bitrate_bps_) >= |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 1451 | kFramedropThreshold * encoder_start_bitrate_bps_) { |
| 1452 | // Reset initial framedrop feature when first real BW estimate arrives. |
| 1453 | // TODO(kthelgason): Update BitrateAllocator to not call OnBitrateUpdated |
| 1454 | // without an actual BW estimate. |
| 1455 | initial_framedrop_ = 0; |
| 1456 | has_seen_first_significant_bwe_change_ = true; |
| 1457 | } |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1458 | |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1459 | uint32_t framerate_fps = GetInputFramerateFps(); |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1460 | frame_dropper_.SetRates((target_bitrate.bps() + 500) / 1000, framerate_fps); |
| 1461 | SetEncoderRates(GetBitrateAllocationAndNotifyObserver(target_bitrate.bps(), |
| 1462 | framerate_fps), |
| 1463 | framerate_fps); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1464 | |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1465 | encoder_start_bitrate_bps_ = target_bitrate.bps() != 0 |
| 1466 | ? target_bitrate.bps() |
| 1467 | : encoder_start_bitrate_bps_; |
| 1468 | bool video_is_suspended = target_bitrate == DataRate::Zero(); |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 1469 | bool video_suspension_changed = video_is_suspended != EncoderPaused(); |
Erik Språng | 610c763 | 2019-03-06 15:37:33 +0100 | [diff] [blame] | 1470 | last_observed_bitrate_bps_ = target_bitrate.bps(); |
Peter Boström | d153a37 | 2015-11-10 15:27:12 +0000 | [diff] [blame] | 1471 | |
sprang | 552c7c7 | 2017-02-13 04:41:45 -0800 | [diff] [blame] | 1472 | if (video_suspension_changed) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1473 | RTC_LOG(LS_INFO) << "Video suspend state changed to: " |
| 1474 | << (video_is_suspended ? "suspended" : "not suspended"); |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1475 | encoder_stats_observer_->OnSuspendChange(video_is_suspended); |
mflodman | 101f250 | 2016-06-09 17:21:19 +0200 | [diff] [blame] | 1476 | } |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1477 | if (video_suspension_changed && !video_is_suspended && pending_frame_ && |
| 1478 | !DropDueToSize(pending_frame_->size())) { |
| 1479 | int64_t pending_time_us = rtc::TimeMicros() - pending_frame_post_time_us_; |
| 1480 | if (pending_time_us < kPendingFrameTimeoutMs * 1000) |
| 1481 | EncodeVideoFrame(*pending_frame_, pending_frame_post_time_us_); |
| 1482 | pending_frame_.reset(); |
| 1483 | } |
| 1484 | } |
| 1485 | |
| 1486 | bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { |
Kári Tristan Helgason | 639602a | 2018-08-02 10:51:40 +0200 | [diff] [blame] | 1487 | if (initial_framedrop_ < kMaxInitialFramedrop && |
Sebastian Jansson | a317705 | 2018-04-10 13:05:49 +0200 | [diff] [blame] | 1488 | encoder_start_bitrate_bps_ > 0) { |
| 1489 | if (encoder_start_bitrate_bps_ < 300000 /* qvga */) { |
| 1490 | return pixel_count > 320 * 240; |
| 1491 | } else if (encoder_start_bitrate_bps_ < 500000 /* vga */) { |
| 1492 | return pixel_count > 640 * 480; |
| 1493 | } |
| 1494 | } |
| 1495 | return false; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1496 | } |
| 1497 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1498 | void VideoStreamEncoder::AdaptDown(AdaptReason reason) { |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 1499 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1500 | AdaptationRequest adaptation_request = { |
| 1501 | last_frame_info_->pixel_count(), |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1502 | encoder_stats_observer_->GetInputFrameRate(), |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1503 | AdaptationRequest::Mode::kAdaptDown}; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1504 | |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1505 | bool downgrade_requested = |
| 1506 | last_adaptation_request_ && |
| 1507 | last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptDown; |
| 1508 | |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1509 | switch (degradation_preference_) { |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1510 | case DegradationPreference::BALANCED: |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1511 | break; |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1512 | case DegradationPreference::MAINTAIN_FRAMERATE: |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1513 | if (downgrade_requested && |
| 1514 | adaptation_request.input_pixel_count_ >= |
| 1515 | last_adaptation_request_->input_pixel_count_) { |
| 1516 | // Don't request lower resolution if the current resolution is not |
| 1517 | // lower than the last time we asked for the resolution to be lowered. |
| 1518 | return; |
| 1519 | } |
| 1520 | break; |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1521 | case DegradationPreference::MAINTAIN_RESOLUTION: |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1522 | if (adaptation_request.framerate_fps_ <= 0 || |
| 1523 | (downgrade_requested && |
| 1524 | adaptation_request.framerate_fps_ < kMinFramerateFps)) { |
| 1525 | // If no input fps estimate available, can't determine how to scale down |
| 1526 | // framerate. Otherwise, don't request lower framerate if we don't have |
| 1527 | // a valid frame rate. Since framerate, unlike resolution, is a measure |
| 1528 | // we have to estimate, and can fluctuate naturally over time, don't |
| 1529 | // make the same kind of limitations as for resolution, but trust the |
| 1530 | // overuse detector to not trigger too often. |
| 1531 | return; |
| 1532 | } |
| 1533 | break; |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1534 | case DegradationPreference::DISABLED: |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1535 | return; |
sprang | 84a3759 | 2017-02-10 07:04:27 -0800 | [diff] [blame] | 1536 | } |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1537 | |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1538 | switch (degradation_preference_) { |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1539 | case DegradationPreference::BALANCED: { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1540 | // Try scale down framerate, if lower. |
| 1541 | int fps = MinFps(last_frame_info_->pixel_count()); |
| 1542 | if (source_proxy_->RestrictFramerate(fps)) { |
| 1543 | GetAdaptCounter().IncrementFramerate(reason); |
| 1544 | break; |
| 1545 | } |
| 1546 | // Scale down resolution. |
Karl Wiberg | 80ba333 | 2018-02-05 10:33:35 +0100 | [diff] [blame] | 1547 | RTC_FALLTHROUGH(); |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1548 | } |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1549 | case DegradationPreference::MAINTAIN_FRAMERATE: { |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1550 | // Scale down resolution. |
Åsa Persson | c3ed630 | 2017-11-16 14:04:52 +0100 | [diff] [blame] | 1551 | bool min_pixels_reached = false; |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1552 | if (!source_proxy_->RequestResolutionLowerThan( |
asapersson | 142fcc9 | 2017-08-17 08:58:54 -0700 | [diff] [blame] | 1553 | adaptation_request.input_pixel_count_, |
Erik Språng | e2fd86a | 2018-10-24 11:32:39 +0200 | [diff] [blame] | 1554 | encoder_->GetEncoderInfo().scaling_settings.min_pixels_per_frame, |
Åsa Persson | c3ed630 | 2017-11-16 14:04:52 +0100 | [diff] [blame] | 1555 | &min_pixels_reached)) { |
| 1556 | if (min_pixels_reached) |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1557 | encoder_stats_observer_->OnMinPixelLimitReached(); |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1558 | return; |
| 1559 | } |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1560 | GetAdaptCounter().IncrementResolution(reason); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1561 | break; |
Åsa Persson | c3ed630 | 2017-11-16 14:04:52 +0100 | [diff] [blame] | 1562 | } |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1563 | case DegradationPreference::MAINTAIN_RESOLUTION: { |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1564 | // Scale down framerate. |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 1565 | const int requested_framerate = source_proxy_->RequestFramerateLowerThan( |
| 1566 | adaptation_request.framerate_fps_); |
| 1567 | if (requested_framerate == -1) |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1568 | return; |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 1569 | RTC_DCHECK_NE(max_framerate_, -1); |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 1570 | overuse_detector_->OnTargetFramerateUpdated( |
| 1571 | std::min(max_framerate_, requested_framerate)); |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1572 | GetAdaptCounter().IncrementFramerate(reason); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1573 | break; |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 1574 | } |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1575 | case DegradationPreference::DISABLED: |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1576 | RTC_NOTREACHED(); |
| 1577 | } |
| 1578 | |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1579 | last_adaptation_request_.emplace(adaptation_request); |
| 1580 | |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1581 | UpdateAdaptationStats(reason); |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1582 | |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1583 | RTC_LOG(LS_INFO) << GetConstAdaptCounter().ToString(); |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1584 | } |
| 1585 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1586 | void VideoStreamEncoder::AdaptUp(AdaptReason reason) { |
perkj | d52063f | 2016-09-07 06:32:18 -0700 | [diff] [blame] | 1587 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1588 | |
| 1589 | const AdaptCounter& adapt_counter = GetConstAdaptCounter(); |
| 1590 | int num_downgrades = adapt_counter.TotalCount(reason); |
| 1591 | if (num_downgrades == 0) |
perkj | 803d97f | 2016-11-01 11:45:46 -0700 | [diff] [blame] | 1592 | return; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1593 | RTC_DCHECK_GT(num_downgrades, 0); |
| 1594 | |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1595 | AdaptationRequest adaptation_request = { |
| 1596 | last_frame_info_->pixel_count(), |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1597 | encoder_stats_observer_->GetInputFrameRate(), |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1598 | AdaptationRequest::Mode::kAdaptUp}; |
| 1599 | |
| 1600 | bool adapt_up_requested = |
| 1601 | last_adaptation_request_ && |
| 1602 | last_adaptation_request_->mode_ == AdaptationRequest::Mode::kAdaptUp; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1603 | |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1604 | if (degradation_preference_ == DegradationPreference::MAINTAIN_FRAMERATE) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1605 | if (adapt_up_requested && |
| 1606 | adaptation_request.input_pixel_count_ <= |
| 1607 | last_adaptation_request_->input_pixel_count_) { |
| 1608 | // Don't request higher resolution if the current resolution is not |
| 1609 | // higher than the last time we asked for the resolution to be higher. |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1610 | return; |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1611 | } |
sprang | b1ca073 | 2017-02-01 08:38:12 -0800 | [diff] [blame] | 1612 | } |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1613 | |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1614 | switch (degradation_preference_) { |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1615 | case DegradationPreference::BALANCED: { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1616 | // Try scale up framerate, if higher. |
| 1617 | int fps = MaxFps(last_frame_info_->pixel_count()); |
| 1618 | if (source_proxy_->IncreaseFramerate(fps)) { |
| 1619 | GetAdaptCounter().DecrementFramerate(reason, fps); |
| 1620 | // Reset framerate in case of fewer fps steps down than up. |
| 1621 | if (adapt_counter.FramerateCount() == 0 && |
| 1622 | fps != std::numeric_limits<int>::max()) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1623 | RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1624 | source_proxy_->IncreaseFramerate(std::numeric_limits<int>::max()); |
| 1625 | } |
| 1626 | break; |
| 1627 | } |
| 1628 | // Scale up resolution. |
Karl Wiberg | 80ba333 | 2018-02-05 10:33:35 +0100 | [diff] [blame] | 1629 | RTC_FALLTHROUGH(); |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1630 | } |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1631 | case DegradationPreference::MAINTAIN_FRAMERATE: { |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1632 | // Scale up resolution. |
| 1633 | int pixel_count = adaptation_request.input_pixel_count_; |
| 1634 | if (adapt_counter.ResolutionCount() == 1) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1635 | RTC_LOG(LS_INFO) << "Removing resolution down-scaling setting."; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1636 | pixel_count = std::numeric_limits<int>::max(); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1637 | } |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1638 | if (!source_proxy_->RequestHigherResolutionThan(pixel_count)) |
| 1639 | return; |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1640 | GetAdaptCounter().DecrementResolution(reason); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1641 | break; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1642 | } |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1643 | case DegradationPreference::MAINTAIN_RESOLUTION: { |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1644 | // Scale up framerate. |
| 1645 | int fps = adaptation_request.framerate_fps_; |
| 1646 | if (adapt_counter.FramerateCount() == 1) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1647 | RTC_LOG(LS_INFO) << "Removing framerate down-scaling setting."; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1648 | fps = std::numeric_limits<int>::max(); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1649 | } |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 1650 | |
| 1651 | const int requested_framerate = |
| 1652 | source_proxy_->RequestHigherFramerateThan(fps); |
| 1653 | if (requested_framerate == -1) { |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 1654 | overuse_detector_->OnTargetFramerateUpdated(max_framerate_); |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1655 | return; |
sprang | fda496a | 2017-06-15 04:21:07 -0700 | [diff] [blame] | 1656 | } |
Niels Möller | 7dc26b7 | 2017-12-06 10:27:48 +0100 | [diff] [blame] | 1657 | overuse_detector_->OnTargetFramerateUpdated( |
| 1658 | std::min(max_framerate_, requested_framerate)); |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1659 | GetAdaptCounter().DecrementFramerate(reason); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1660 | break; |
asapersson | 1387476 | 2017-06-07 00:01:02 -0700 | [diff] [blame] | 1661 | } |
Taylor Brandstetter | 49fcc10 | 2018-05-16 14:20:41 -0700 | [diff] [blame] | 1662 | case DegradationPreference::DISABLED: |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1663 | return; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1664 | } |
| 1665 | |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1666 | last_adaptation_request_.emplace(adaptation_request); |
| 1667 | |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1668 | UpdateAdaptationStats(reason); |
| 1669 | |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1670 | RTC_LOG(LS_INFO) << adapt_counter.ToString(); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1671 | } |
| 1672 | |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1673 | // TODO(nisse): Delete, once AdaptReason and AdaptationReason are merged. |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1674 | void VideoStreamEncoder::UpdateAdaptationStats(AdaptReason reason) { |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1675 | switch (reason) { |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1676 | case kCpu: |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1677 | encoder_stats_observer_->OnAdaptationChanged( |
| 1678 | VideoStreamEncoderObserver::AdaptationReason::kCpu, |
| 1679 | GetActiveCounts(kCpu), GetActiveCounts(kQuality)); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1680 | break; |
| 1681 | case kQuality: |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1682 | encoder_stats_observer_->OnAdaptationChanged( |
| 1683 | VideoStreamEncoderObserver::AdaptationReason::kQuality, |
| 1684 | GetActiveCounts(kCpu), GetActiveCounts(kQuality)); |
asapersson | d0de295 | 2017-04-21 01:47:31 -0700 | [diff] [blame] | 1685 | break; |
| 1686 | } |
perkj | 26091b1 | 2016-09-01 01:17:40 -0700 | [diff] [blame] | 1687 | } |
| 1688 | |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1689 | VideoStreamEncoderObserver::AdaptationSteps VideoStreamEncoder::GetActiveCounts( |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1690 | AdaptReason reason) { |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1691 | VideoStreamEncoderObserver::AdaptationSteps counts = |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1692 | GetConstAdaptCounter().Counts(reason); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1693 | switch (reason) { |
| 1694 | case kCpu: |
| 1695 | if (!IsFramerateScalingEnabled(degradation_preference_)) |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1696 | counts.num_framerate_reductions = absl::nullopt; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1697 | if (!IsResolutionScalingEnabled(degradation_preference_)) |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1698 | counts.num_resolution_reductions = absl::nullopt; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1699 | break; |
| 1700 | case kQuality: |
| 1701 | if (!IsFramerateScalingEnabled(degradation_preference_) || |
| 1702 | !quality_scaler_) { |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1703 | counts.num_framerate_reductions = absl::nullopt; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1704 | } |
| 1705 | if (!IsResolutionScalingEnabled(degradation_preference_) || |
| 1706 | !quality_scaler_) { |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1707 | counts.num_resolution_reductions = absl::nullopt; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1708 | } |
| 1709 | break; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1710 | } |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1711 | return counts; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1712 | } |
| 1713 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1714 | VideoStreamEncoder::AdaptCounter& VideoStreamEncoder::GetAdaptCounter() { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1715 | return adapt_counters_[degradation_preference_]; |
| 1716 | } |
| 1717 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1718 | const VideoStreamEncoder::AdaptCounter& |
| 1719 | VideoStreamEncoder::GetConstAdaptCounter() { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1720 | return adapt_counters_[degradation_preference_]; |
| 1721 | } |
| 1722 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1723 | void VideoStreamEncoder::RunPostEncode(EncodedImage encoded_image, |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1724 | int64_t time_sent_us, |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1725 | int temporal_index) { |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1726 | if (!encoder_queue_.IsCurrent()) { |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1727 | encoder_queue_.PostTask( |
| 1728 | [this, encoded_image, time_sent_us, temporal_index] { |
| 1729 | RunPostEncode(encoded_image, time_sent_us, temporal_index); |
| 1730 | }); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1731 | return; |
| 1732 | } |
| 1733 | |
| 1734 | RTC_DCHECK_RUN_ON(&encoder_queue_); |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1735 | |
| 1736 | absl::optional<int> encode_duration_us; |
| 1737 | if (encoded_image.timing_.flags != VideoSendTiming::kInvalid) { |
| 1738 | encode_duration_us = |
| 1739 | // TODO(nisse): Maybe use capture_time_ms_ rather than encode_start_ms_? |
| 1740 | rtc::kNumMicrosecsPerMillisec * |
| 1741 | (encoded_image.timing_.encode_finish_ms - |
| 1742 | encoded_image.timing_.encode_start_ms); |
| 1743 | } |
| 1744 | |
| 1745 | // Run post encode tasks, such as overuse detection and frame rate/drop |
| 1746 | // stats for internal encoders. |
| 1747 | const size_t frame_size = encoded_image.size(); |
Niels Möller | 87e2d78 | 2019-03-07 10:18:23 +0100 | [diff] [blame] | 1748 | const bool keyframe = |
| 1749 | encoded_image._frameType == VideoFrameType::kVideoFrameKey; |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1750 | |
| 1751 | if (frame_size > 0) { |
| 1752 | frame_dropper_.Fill(frame_size, !keyframe); |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1753 | } |
| 1754 | |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1755 | if (HasInternalSource()) { |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1756 | // Update frame dropper after the fact for internal sources. |
| 1757 | input_framerate_.Update(1u, clock_->TimeInMilliseconds()); |
| 1758 | frame_dropper_.Leak(GetInputFramerateFps()); |
| 1759 | // Signal to encoder to drop next frame. |
| 1760 | if (frame_dropper_.DropFrame()) { |
| 1761 | pending_frame_drops_.fetch_add(1); |
| 1762 | } |
| 1763 | } |
| 1764 | |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1765 | overuse_detector_->FrameSent( |
| 1766 | encoded_image.Timestamp(), time_sent_us, |
| 1767 | encoded_image.capture_time_ms_ * rtc::kNumMicrosecsPerMillisec, |
| 1768 | encode_duration_us); |
| 1769 | if (quality_scaler_ && encoded_image.qp_ >= 0) |
Sebastian Jansson | b678940 | 2019-03-01 15:40:49 +0100 | [diff] [blame] | 1770 | quality_scaler_->ReportQp(encoded_image.qp_, time_sent_us); |
Erik Språng | 7ca375c | 2019-02-06 16:20:17 +0100 | [diff] [blame] | 1771 | if (bitrate_adjuster_) { |
| 1772 | bitrate_adjuster_->OnEncodedFrame(encoded_image, temporal_index); |
| 1773 | } |
Niels Möller | 6bb5ab9 | 2019-01-11 11:11:10 +0100 | [diff] [blame] | 1774 | } |
| 1775 | |
Erik Språng | d7329ca | 2019-02-21 21:19:53 +0100 | [diff] [blame] | 1776 | bool VideoStreamEncoder::HasInternalSource() const { |
| 1777 | // TODO(sprang): Checking both info from encoder and from encoder factory |
| 1778 | // until we have deprecated and removed the encoder factory info. |
| 1779 | return codec_info_.has_internal_source || encoder_info_.has_internal_source; |
| 1780 | } |
| 1781 | |
Erik Språng | 6a7baa7 | 2019-02-26 18:31:00 +0100 | [diff] [blame] | 1782 | void VideoStreamEncoder::ReleaseEncoder() { |
| 1783 | if (!encoder_ || !encoder_initialized_) { |
| 1784 | return; |
| 1785 | } |
| 1786 | encoder_->Release(); |
| 1787 | encoder_initialized_ = false; |
| 1788 | TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release"); |
| 1789 | } |
| 1790 | |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1791 | // Class holding adaptation information. |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1792 | VideoStreamEncoder::AdaptCounter::AdaptCounter() { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1793 | fps_counters_.resize(kScaleReasonSize); |
| 1794 | resolution_counters_.resize(kScaleReasonSize); |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1795 | static_assert(kScaleReasonSize == 2, "Update MoveCount."); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1796 | } |
| 1797 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1798 | VideoStreamEncoder::AdaptCounter::~AdaptCounter() {} |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1799 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1800 | std::string VideoStreamEncoder::AdaptCounter::ToString() const { |
Jonas Olsson | 366a50c | 2018-09-06 13:41:30 +0200 | [diff] [blame] | 1801 | rtc::StringBuilder ss; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1802 | ss << "Downgrade counts: fps: {" << ToString(fps_counters_); |
| 1803 | ss << "}, resolution: {" << ToString(resolution_counters_) << "}"; |
Jonas Olsson | 84df1c7 | 2018-09-14 16:59:32 +0200 | [diff] [blame] | 1804 | return ss.Release(); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1805 | } |
| 1806 | |
Niels Möller | 213618e | 2018-07-24 09:29:58 +0200 | [diff] [blame] | 1807 | VideoStreamEncoderObserver::AdaptationSteps |
| 1808 | VideoStreamEncoder::AdaptCounter::Counts(int reason) const { |
| 1809 | VideoStreamEncoderObserver::AdaptationSteps counts; |
| 1810 | counts.num_framerate_reductions = fps_counters_[reason]; |
| 1811 | counts.num_resolution_reductions = resolution_counters_[reason]; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1812 | return counts; |
| 1813 | } |
| 1814 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1815 | void VideoStreamEncoder::AdaptCounter::IncrementFramerate(int reason) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1816 | ++(fps_counters_[reason]); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1817 | } |
| 1818 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1819 | void VideoStreamEncoder::AdaptCounter::IncrementResolution(int reason) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1820 | ++(resolution_counters_[reason]); |
| 1821 | } |
| 1822 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1823 | void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1824 | if (fps_counters_[reason] == 0) { |
| 1825 | // Balanced mode: Adapt up is in a different order, switch reason. |
| 1826 | // E.g. framerate adapt down: quality (2), framerate adapt up: cpu (3). |
| 1827 | // 1. Down resolution (cpu): res={quality:0,cpu:1}, fps={quality:0,cpu:0} |
| 1828 | // 2. Down fps (quality): res={quality:0,cpu:1}, fps={quality:1,cpu:0} |
| 1829 | // 3. Up fps (cpu): res={quality:1,cpu:0}, fps={quality:0,cpu:0} |
| 1830 | // 4. Up resolution (quality): res={quality:0,cpu:0}, fps={quality:0,cpu:0} |
| 1831 | RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; |
| 1832 | RTC_DCHECK_GT(FramerateCount(), 0) << "Framerate not downgraded."; |
| 1833 | MoveCount(&resolution_counters_, reason); |
| 1834 | MoveCount(&fps_counters_, (reason + 1) % kScaleReasonSize); |
| 1835 | } |
| 1836 | --(fps_counters_[reason]); |
| 1837 | RTC_DCHECK_GE(fps_counters_[reason], 0); |
| 1838 | } |
| 1839 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1840 | void VideoStreamEncoder::AdaptCounter::DecrementResolution(int reason) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1841 | if (resolution_counters_[reason] == 0) { |
| 1842 | // Balanced mode: Adapt up is in a different order, switch reason. |
| 1843 | RTC_DCHECK_GT(TotalCount(reason), 0) << "No downgrade for reason."; |
| 1844 | RTC_DCHECK_GT(ResolutionCount(), 0) << "Resolution not downgraded."; |
| 1845 | MoveCount(&fps_counters_, reason); |
| 1846 | MoveCount(&resolution_counters_, (reason + 1) % kScaleReasonSize); |
| 1847 | } |
| 1848 | --(resolution_counters_[reason]); |
| 1849 | RTC_DCHECK_GE(resolution_counters_[reason], 0); |
| 1850 | } |
| 1851 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1852 | void VideoStreamEncoder::AdaptCounter::DecrementFramerate(int reason, |
| 1853 | int cur_fps) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1854 | DecrementFramerate(reason); |
| 1855 | // Reset if at max fps (i.e. in case of fewer steps up than down). |
| 1856 | if (cur_fps == std::numeric_limits<int>::max()) |
| 1857 | std::fill(fps_counters_.begin(), fps_counters_.end(), 0); |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1858 | } |
| 1859 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1860 | int VideoStreamEncoder::AdaptCounter::FramerateCount() const { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1861 | return Count(fps_counters_); |
| 1862 | } |
| 1863 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1864 | int VideoStreamEncoder::AdaptCounter::ResolutionCount() const { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1865 | return Count(resolution_counters_); |
| 1866 | } |
| 1867 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1868 | int VideoStreamEncoder::AdaptCounter::FramerateCount(int reason) const { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1869 | return fps_counters_[reason]; |
| 1870 | } |
| 1871 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1872 | int VideoStreamEncoder::AdaptCounter::ResolutionCount(int reason) const { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1873 | return resolution_counters_[reason]; |
| 1874 | } |
| 1875 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1876 | int VideoStreamEncoder::AdaptCounter::TotalCount(int reason) const { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1877 | return FramerateCount(reason) + ResolutionCount(reason); |
| 1878 | } |
| 1879 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1880 | int VideoStreamEncoder::AdaptCounter::Count( |
| 1881 | const std::vector<int>& counters) const { |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1882 | return std::accumulate(counters.begin(), counters.end(), 0); |
| 1883 | } |
| 1884 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1885 | void VideoStreamEncoder::AdaptCounter::MoveCount(std::vector<int>* counters, |
| 1886 | int from_reason) { |
asapersson | f7e294d | 2017-06-13 23:25:22 -0700 | [diff] [blame] | 1887 | int to_reason = (from_reason + 1) % kScaleReasonSize; |
| 1888 | ++((*counters)[to_reason]); |
| 1889 | --((*counters)[from_reason]); |
| 1890 | } |
| 1891 | |
mflodman | cc3d442 | 2017-08-03 08:27:51 -0700 | [diff] [blame] | 1892 | std::string VideoStreamEncoder::AdaptCounter::ToString( |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1893 | const std::vector<int>& counters) const { |
Jonas Olsson | 366a50c | 2018-09-06 13:41:30 +0200 | [diff] [blame] | 1894 | rtc::StringBuilder ss; |
asapersson | 09f0561 | 2017-05-15 23:40:18 -0700 | [diff] [blame] | 1895 | for (size_t reason = 0; reason < kScaleReasonSize; ++reason) { |
| 1896 | ss << (reason ? " cpu" : "quality") << ":" << counters[reason]; |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1897 | } |
Jonas Olsson | 84df1c7 | 2018-09-14 16:59:32 +0200 | [diff] [blame] | 1898 | return ss.Release(); |
sprang | c5d62e2 | 2017-04-02 23:53:04 -0700 | [diff] [blame] | 1899 | } |
| 1900 | |
mflodman@webrtc.org | 84d1783 | 2011-12-01 17:02:23 +0000 | [diff] [blame] | 1901 | } // namespace webrtc |