blob: c8ea6ca1c4282276359253d9a8d012d26163206f [file] [log] [blame]
pbos@webrtc.org9115cde2014-12-09 10:36:40 +00001/* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
Ilya Nikolaevskiybf352982017-10-02 10:08:25 +02002 *
3 * Use of this source code is governed by a BSD-style license
4 * that can be found in the LICENSE file in the root of the source
5 * tree. An additional intellectual property rights grant can be found
6 * in the file PATENTS. All contributing project authors may
7 * be found in the AUTHORS file in the root of the source tree.
8 */
pbos@webrtc.org9115cde2014-12-09 10:36:40 +00009
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020010#include "modules/video_coding/codecs/vp8/screenshare_layers.h"
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000011
12#include <stdlib.h>
13
philipelcce46fc2015-12-21 03:04:49 -080014#include <algorithm>
Ilya Nikolaevskiybf352982017-10-02 10:08:25 +020015#include <memory>
philipelcce46fc2015-12-21 03:04:49 -080016
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "modules/video_coding/include/video_codec_interface.h"
Elad Alon819661a2019-01-29 17:17:09 +010018#include "rtc_base/arraysize.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "rtc_base/checks.h"
Ilya Nikolaevskiy58662912017-10-04 15:07:09 +020020#include "rtc_base/logging.h"
Sebastian Janssone64a6882019-03-01 18:04:07 +010021#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "system_wrappers/include/metrics.h"
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000023
24namespace webrtc {
Erik Språng59021ba2018-10-03 11:05:16 +020025namespace {
Elad Alon411b49b2019-01-29 14:05:55 +010026using BufferFlags = Vp8FrameConfig::BufferFlags;
27
28constexpr BufferFlags kNone = Vp8FrameConfig::BufferFlags::kNone;
29constexpr BufferFlags kReference = Vp8FrameConfig::BufferFlags::kReference;
30constexpr BufferFlags kUpdate = Vp8FrameConfig::BufferFlags::kUpdate;
31constexpr BufferFlags kReferenceAndUpdate =
32 Vp8FrameConfig::BufferFlags::kReferenceAndUpdate;
33
34constexpr int kOneSecond90Khz = 90000;
35constexpr int kMinTimeBetweenSyncs = kOneSecond90Khz * 2;
36constexpr int kMaxTimeBetweenSyncs = kOneSecond90Khz * 4;
37constexpr int kQpDeltaThresholdForSync = 8;
38constexpr int kMinBitrateKbpsForQpBoost = 500;
Erik Språng59021ba2018-10-03 11:05:16 +020039} // namespace
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000040
sprang@webrtc.org70f74f32014-12-17 10:57:10 +000041const double ScreenshareLayers::kMaxTL0FpsReduction = 2.5;
42const double ScreenshareLayers::kAcceptableTargetOvershoot = 2.0;
43
sprang429600d2017-01-26 06:12:26 -080044constexpr int ScreenshareLayers::kMaxNumTemporalLayers;
45
sprangafe1f742016-04-12 02:45:13 -070046// Always emit a frame with certain interval, even if bitrate targets have
sprang916170a2017-05-23 07:47:55 -070047// been exceeded. This prevents needless keyframe requests.
sprang6c4bbfa2017-05-30 10:08:23 -070048const int ScreenshareLayers::kMaxFrameIntervalMs = 2750;
sprangafe1f742016-04-12 02:45:13 -070049
Sebastian Janssone64a6882019-03-01 18:04:07 +010050ScreenshareLayers::ScreenshareLayers(int num_temporal_layers)
51 : number_of_temporal_layers_(
sprang429600d2017-01-26 06:12:26 -080052 std::min(kMaxNumTemporalLayers, num_temporal_layers)),
Erik Språng2c4c9142015-06-24 11:24:44 +020053 active_layer_(-1),
54 last_timestamp_(-1),
55 last_sync_timestamp_(-1),
sprangafe1f742016-04-12 02:45:13 -070056 last_emitted_tl0_timestamp_(-1),
Erik Språng27a457d2018-01-12 11:00:21 +010057 last_frame_time_ms_(-1),
Erik Språng2c4c9142015-06-24 11:24:44 +020058 max_debt_bytes_(0),
sprangac4a90d2016-12-28 05:58:07 -080059 encode_framerate_(1000.0f, 1000.0f), // 1 second window, second scale.
Erik Språng59021ba2018-10-03 11:05:16 +020060 bitrate_updated_(false),
61 checker_(TemporalLayersChecker::CreateTemporalLayersChecker(
Erik Språng4529fbc2018-10-12 10:30:31 +020062 Vp8TemporalLayersType::kBitrateDynamic,
Erik Språng59021ba2018-10-03 11:05:16 +020063 num_temporal_layers)) {
sprang429600d2017-01-26 06:12:26 -080064 RTC_CHECK_GT(number_of_temporal_layers_, 0);
65 RTC_CHECK_LE(number_of_temporal_layers_, kMaxNumTemporalLayers);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000066}
67
sprangb0fdfea2016-03-01 05:51:16 -080068ScreenshareLayers::~ScreenshareLayers() {
69 UpdateHistograms();
70}
71
Elad Alonfb087812019-05-02 23:25:34 +020072void ScreenshareLayers::SetQpLimits(size_t stream_index,
73 int min_qp,
74 int max_qp) {
75 RTC_DCHECK_LT(stream_index, StreamCount());
76 // 0 < min_qp <= max_qp
77 RTC_DCHECK_LT(0, min_qp);
78 RTC_DCHECK_LE(min_qp, max_qp);
79
80 RTC_DCHECK_EQ(min_qp_.has_value(), max_qp_.has_value());
81 if (!min_qp_.has_value()) {
82 min_qp_ = min_qp;
83 max_qp_ = max_qp;
84 } else {
85 RTC_DCHECK_EQ(min_qp, min_qp_.value());
86 RTC_DCHECK_EQ(max_qp, max_qp_.value());
87 }
88}
89
Elad Aloncde8ab22019-03-20 11:56:20 +010090size_t ScreenshareLayers::StreamCount() const {
91 return 1;
92}
93
94bool ScreenshareLayers::SupportsEncoderFrameDropping(
95 size_t stream_index) const {
96 RTC_DCHECK_LT(stream_index, StreamCount());
Erik Språngfb2a66a2018-09-10 10:48:01 +020097 // Frame dropping is handled internally by this class.
98 return false;
99}
100
Elad Alon979c4422019-04-17 12:53:08 +0200101Vp8FrameConfig ScreenshareLayers::NextFrameConfig(size_t stream_index,
102 uint32_t timestamp) {
Elad Aloncde8ab22019-03-20 11:56:20 +0100103 RTC_DCHECK_LT(stream_index, StreamCount());
104
Erik Språng59021ba2018-10-03 11:05:16 +0200105 auto it = pending_frame_configs_.find(timestamp);
106 if (it != pending_frame_configs_.end()) {
107 // Drop and re-encode, reuse the previous config.
philipelb19f27a2019-03-28 16:09:52 +0100108 return it->second.frame_config;
Erik Språng59021ba2018-10-03 11:05:16 +0200109 }
110
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000111 if (number_of_temporal_layers_ <= 1) {
112 // No flags needed for 1 layer screenshare.
Peter Boström1436c832017-03-27 15:01:49 -0400113 // TODO(pbos): Consider updating only last, and not all buffers.
philipelb19f27a2019-03-28 16:09:52 +0100114 DependencyInfo dependency_info{
115 "S", {kReferenceAndUpdate, kReferenceAndUpdate, kReferenceAndUpdate}};
116 pending_frame_configs_[timestamp] = dependency_info;
117 return dependency_info.frame_config;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000118 }
Erik Språng2c4c9142015-06-24 11:24:44 +0200119
Sebastian Janssone64a6882019-03-01 18:04:07 +0100120 const int64_t now_ms = rtc::TimeMillis();
sprangb0fdfea2016-03-01 05:51:16 -0800121
Erik Språng2c4c9142015-06-24 11:24:44 +0200122 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(timestamp);
sprang916170a2017-05-23 07:47:55 -0700123 int64_t ts_diff;
124 if (last_timestamp_ == -1) {
125 ts_diff = kOneSecond90Khz / capture_framerate_.value_or(*target_framerate_);
126 } else {
127 ts_diff = unwrapped_timestamp - last_timestamp_;
128 }
Erik Språng27a457d2018-01-12 11:00:21 +0100129
130 if (target_framerate_) {
131 // If input frame rate exceeds target frame rate, either over a one second
132 // averaging window, or if frame interval is below 90% of desired value,
133 // drop frame.
Erik Språng27a457d2018-01-12 11:00:21 +0100134 if (encode_framerate_.Rate(now_ms).value_or(0) > *target_framerate_)
Elad Alon411b49b2019-01-29 14:05:55 +0100135 return Vp8FrameConfig(kNone, kNone, kNone);
Erik Språng27a457d2018-01-12 11:00:21 +0100136
Erik Språngdb9e9d52018-01-22 09:23:11 -0800137 // Primarily check if frame interval is too short using frame timestamps,
138 // as if they are correct they won't be affected by queuing in webrtc.
139 const int64_t expected_frame_interval_90khz =
140 kOneSecond90Khz / *target_framerate_;
141 if (last_timestamp_ != -1 && ts_diff > 0) {
142 if (ts_diff < 85 * expected_frame_interval_90khz / 100) {
Elad Alon411b49b2019-01-29 14:05:55 +0100143 return Vp8FrameConfig(kNone, kNone, kNone);
Erik Språngdb9e9d52018-01-22 09:23:11 -0800144 }
145 } else {
146 // Timestamps looks off, use realtime clock here instead.
147 const int64_t expected_frame_interval_ms = 1000 / *target_framerate_;
148 if (last_frame_time_ms_ != -1 &&
149 now_ms - last_frame_time_ms_ <
150 (85 * expected_frame_interval_ms) / 100) {
Elad Alon411b49b2019-01-29 14:05:55 +0100151 return Vp8FrameConfig(kNone, kNone, kNone);
Erik Språngdb9e9d52018-01-22 09:23:11 -0800152 }
Erik Språng27a457d2018-01-12 11:00:21 +0100153 }
154 }
155
156 if (stats_.first_frame_time_ms_ == -1)
157 stats_.first_frame_time_ms_ = now_ms;
158
sprang916170a2017-05-23 07:47:55 -0700159 // Make sure both frame droppers leak out bits.
160 layers_[0].UpdateDebt(ts_diff / 90);
161 layers_[1].UpdateDebt(ts_diff / 90);
162 last_timestamp_ = timestamp;
Erik Språng27a457d2018-01-12 11:00:21 +0100163 last_frame_time_ms_ = now_ms;
sprang916170a2017-05-23 07:47:55 -0700164
165 TemporalLayerState layer_state = TemporalLayerState::kDrop;
166
Erik Språng2c4c9142015-06-24 11:24:44 +0200167 if (active_layer_ == -1 ||
168 layers_[active_layer_].state != TemporalLayer::State::kDropped) {
sprangafe1f742016-04-12 02:45:13 -0700169 if (last_emitted_tl0_timestamp_ != -1 &&
170 (unwrapped_timestamp - last_emitted_tl0_timestamp_) / 90 >
171 kMaxFrameIntervalMs) {
172 // Too long time has passed since the last frame was emitted, cancel
173 // enough debt to allow a single frame.
174 layers_[0].debt_bytes_ = max_debt_bytes_ - 1;
175 }
Erik Språng2c4c9142015-06-24 11:24:44 +0200176 if (layers_[0].debt_bytes_ > max_debt_bytes_) {
177 // Must drop TL0, encode TL1 instead.
178 if (layers_[1].debt_bytes_ > max_debt_bytes_) {
179 // Must drop both TL0 and TL1.
180 active_layer_ = -1;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000181 } else {
Erik Språng2c4c9142015-06-24 11:24:44 +0200182 active_layer_ = 1;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000183 }
Erik Språng2c4c9142015-06-24 11:24:44 +0200184 } else {
185 active_layer_ = 0;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000186 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000187 }
Erik Språng2c4c9142015-06-24 11:24:44 +0200188
189 switch (active_layer_) {
190 case 0:
sprang916170a2017-05-23 07:47:55 -0700191 layer_state = TemporalLayerState::kTl0;
sprangafe1f742016-04-12 02:45:13 -0700192 last_emitted_tl0_timestamp_ = unwrapped_timestamp;
Erik Språng2c4c9142015-06-24 11:24:44 +0200193 break;
194 case 1:
sprangf03ea042017-07-13 03:53:51 -0700195 if (layers_[1].state != TemporalLayer::State::kDropped) {
Erik Språngb75d6b82018-08-13 16:05:33 +0200196 if (TimeToSync(unwrapped_timestamp) ||
197 layers_[1].state == TemporalLayer::State::kKeyFrame) {
sprangf03ea042017-07-13 03:53:51 -0700198 last_sync_timestamp_ = unwrapped_timestamp;
199 layer_state = TemporalLayerState::kTl1Sync;
200 } else {
201 layer_state = TemporalLayerState::kTl1;
202 }
Erik Språng2c4c9142015-06-24 11:24:44 +0200203 } else {
sprangf03ea042017-07-13 03:53:51 -0700204 layer_state = last_sync_timestamp_ == unwrapped_timestamp
205 ? TemporalLayerState::kTl1Sync
206 : TemporalLayerState::kTl1;
Erik Språng2c4c9142015-06-24 11:24:44 +0200207 }
208 break;
209 case -1:
sprang916170a2017-05-23 07:47:55 -0700210 layer_state = TemporalLayerState::kDrop;
sprangb0fdfea2016-03-01 05:51:16 -0800211 ++stats_.num_dropped_frames_;
Erik Språng2c4c9142015-06-24 11:24:44 +0200212 break;
213 default:
Erik Språng2c4c9142015-06-24 11:24:44 +0200214 RTC_NOTREACHED();
215 }
216
philipelb19f27a2019-03-28 16:09:52 +0100217 DependencyInfo dependency_info;
Peter Boström1436c832017-03-27 15:01:49 -0400218 // TODO(pbos): Consider referencing but not updating the 'alt' buffer for all
219 // layers.
220 switch (layer_state) {
sprang916170a2017-05-23 07:47:55 -0700221 case TemporalLayerState::kDrop:
philipelb19f27a2019-03-28 16:09:52 +0100222 dependency_info = {"", {kNone, kNone, kNone}};
pbos51f083c2017-05-04 06:39:04 -0700223 break;
sprang916170a2017-05-23 07:47:55 -0700224 case TemporalLayerState::kTl0:
Peter Boström1436c832017-03-27 15:01:49 -0400225 // TL0 only references and updates 'last'.
philipelb19f27a2019-03-28 16:09:52 +0100226 dependency_info = {"SS", {kReferenceAndUpdate, kNone, kNone}};
227 dependency_info.frame_config.packetizer_temporal_idx = 0;
pbos51f083c2017-05-04 06:39:04 -0700228 break;
sprang916170a2017-05-23 07:47:55 -0700229 case TemporalLayerState::kTl1:
Peter Boström1436c832017-03-27 15:01:49 -0400230 // TL1 references both 'last' and 'golden' but only updates 'golden'.
philipelb19f27a2019-03-28 16:09:52 +0100231 dependency_info = {"-R", {kReference, kReferenceAndUpdate, kNone}};
232 dependency_info.frame_config.packetizer_temporal_idx = 1;
pbos51f083c2017-05-04 06:39:04 -0700233 break;
sprang916170a2017-05-23 07:47:55 -0700234 case TemporalLayerState::kTl1Sync:
Peter Boström1436c832017-03-27 15:01:49 -0400235 // Predict from only TL0 to allow participants to switch to the high
236 // bitrate stream. Updates 'golden' so that TL1 can continue to refer to
237 // and update 'golden' from this point on.
philipelb19f27a2019-03-28 16:09:52 +0100238 dependency_info = {"-S", {kReference, kUpdate, kNone}};
239 dependency_info.frame_config.packetizer_temporal_idx = 1;
240 dependency_info.frame_config.layer_sync = true;
pbos51f083c2017-05-04 06:39:04 -0700241 break;
Peter Boström1436c832017-03-27 15:01:49 -0400242 }
pbos51f083c2017-05-04 06:39:04 -0700243
philipelb19f27a2019-03-28 16:09:52 +0100244 pending_frame_configs_[timestamp] = dependency_info;
245 return dependency_info.frame_config;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000246}
247
Erik Språngbb60a3a2018-03-19 18:25:10 +0100248void ScreenshareLayers::OnRatesUpdated(
Elad Aloncde8ab22019-03-20 11:56:20 +0100249 size_t stream_index,
Erik Språngbb60a3a2018-03-19 18:25:10 +0100250 const std::vector<uint32_t>& bitrates_bps,
251 int framerate_fps) {
Elad Aloncde8ab22019-03-20 11:56:20 +0100252 RTC_DCHECK_LT(stream_index, StreamCount());
Erik Språngbb60a3a2018-03-19 18:25:10 +0100253 RTC_DCHECK_GT(framerate_fps, 0);
254 RTC_DCHECK_GE(bitrates_bps.size(), 1);
255 RTC_DCHECK_LE(bitrates_bps.size(), 2);
256
257 // |bitrates_bps| uses individual rates per layer, but we want to use the
258 // accumulated rate here.
259 uint32_t tl0_kbps = bitrates_bps[0] / 1000;
260 uint32_t tl1_kbps = tl0_kbps;
261 if (bitrates_bps.size() > 1) {
262 tl1_kbps += bitrates_bps[1] / 1000;
263 }
264
sprangac4a90d2016-12-28 05:58:07 -0800265 if (!target_framerate_) {
Erik Språngbb60a3a2018-03-19 18:25:10 +0100266 // First OnRatesUpdated() is called during construction, with the
267 // configured targets as parameters.
268 target_framerate_ = framerate_fps;
sprang0ad0de62017-01-11 05:01:32 -0800269 capture_framerate_ = target_framerate_;
sprangac4a90d2016-12-28 05:58:07 -0800270 bitrate_updated_ = true;
271 } else {
Erik Språnge624d072018-04-11 16:47:27 +0200272 if ((capture_framerate_ &&
273 framerate_fps != static_cast<int>(*capture_framerate_)) ||
274 (tl0_kbps != layers_[0].target_rate_kbps_) ||
275 (tl1_kbps != layers_[1].target_rate_kbps_)) {
276 bitrate_updated_ = true;
277 }
Erik Språngbb60a3a2018-03-19 18:25:10 +0100278
279 if (framerate_fps < 0) {
sprang0ad0de62017-01-11 05:01:32 -0800280 capture_framerate_.reset();
sprangac4a90d2016-12-28 05:58:07 -0800281 } else {
Erik Språngbb60a3a2018-03-19 18:25:10 +0100282 capture_framerate_ = framerate_fps;
sprangac4a90d2016-12-28 05:58:07 -0800283 }
284 }
285
Erik Språngbb60a3a2018-03-19 18:25:10 +0100286 layers_[0].target_rate_kbps_ = tl0_kbps;
287 layers_[1].target_rate_kbps_ = tl1_kbps;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000288}
289
Elad Aloncde8ab22019-03-20 11:56:20 +0100290void ScreenshareLayers::OnEncodeDone(size_t stream_index,
291 uint32_t rtp_timestamp,
Erik Språng59021ba2018-10-03 11:05:16 +0200292 size_t size_bytes,
293 bool is_keyframe,
294 int qp,
philipel9df33532019-03-04 16:37:50 +0100295 CodecSpecificInfo* info) {
Elad Aloncde8ab22019-03-20 11:56:20 +0100296 RTC_DCHECK_LT(stream_index, StreamCount());
297
Erik Språng59021ba2018-10-03 11:05:16 +0200298 if (size_bytes == 0) {
Elad Alon6796ec22019-04-15 10:07:50 +0200299 RTC_LOG(LS_WARNING) << "Empty frame; treating as dropped.";
300 OnFrameDropped(stream_index, rtp_timestamp);
Erik Språng2c4c9142015-06-24 11:24:44 +0200301 return;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000302 }
sprangef7228c2015-08-05 02:01:29 -0700303
philipelb19f27a2019-03-28 16:09:52 +0100304 absl::optional<DependencyInfo> dependency_info;
Erik Språng59021ba2018-10-03 11:05:16 +0200305 auto it = pending_frame_configs_.find(rtp_timestamp);
306 if (it != pending_frame_configs_.end()) {
philipelb19f27a2019-03-28 16:09:52 +0100307 dependency_info = it->second;
Erik Språng59021ba2018-10-03 11:05:16 +0200308 pending_frame_configs_.erase(it);
309
310 if (checker_) {
philipelb19f27a2019-03-28 16:09:52 +0100311 RTC_DCHECK(checker_->CheckTemporalConfig(is_keyframe,
312 dependency_info->frame_config));
Erik Språng59021ba2018-10-03 11:05:16 +0200313 }
314 }
315
philipel9df33532019-03-04 16:37:50 +0100316 CodecSpecificInfoVP8& vp8_info = info->codecSpecific.VP8;
philipelb19f27a2019-03-28 16:09:52 +0100317 GenericFrameInfo& generic_frame_info = info->generic_frame_info.emplace();
318
Erik Språng59021ba2018-10-03 11:05:16 +0200319 if (number_of_temporal_layers_ == 1) {
philipel9df33532019-03-04 16:37:50 +0100320 vp8_info.temporalIdx = kNoTemporalIdx;
321 vp8_info.layerSync = false;
philipelb19f27a2019-03-28 16:09:52 +0100322 generic_frame_info.decode_target_indications =
323 GenericFrameInfo::DecodeTargetInfo("S");
philipelda5aa4d2019-04-26 13:37:37 +0200324 generic_frame_info.encoder_buffers.emplace_back(
325 0, /*referenced=*/!is_keyframe, /*updated=*/true);
Erik Språng59021ba2018-10-03 11:05:16 +0200326 } else {
327 int64_t unwrapped_timestamp = time_wrap_handler_.Unwrap(rtp_timestamp);
philipelb19f27a2019-03-28 16:09:52 +0100328 if (dependency_info) {
329 vp8_info.temporalIdx =
330 dependency_info->frame_config.packetizer_temporal_idx;
331 vp8_info.layerSync = dependency_info->frame_config.layer_sync;
332 generic_frame_info.decode_target_indications =
333 dependency_info->decode_target_indications;
Erik Språng59021ba2018-10-03 11:05:16 +0200334 } else {
Elad Alon819661a2019-01-29 17:17:09 +0100335 RTC_DCHECK(is_keyframe);
philipelb19f27a2019-03-28 16:09:52 +0100336 generic_frame_info.decode_target_indications =
337 GenericFrameInfo::DecodeTargetInfo("SS");
Erik Språng59021ba2018-10-03 11:05:16 +0200338 }
Elad Alon819661a2019-01-29 17:17:09 +0100339
Erik Språng59021ba2018-10-03 11:05:16 +0200340 if (is_keyframe) {
philipel9df33532019-03-04 16:37:50 +0100341 vp8_info.temporalIdx = 0;
Erik Språng59021ba2018-10-03 11:05:16 +0200342 last_sync_timestamp_ = unwrapped_timestamp;
philipel9df33532019-03-04 16:37:50 +0100343 vp8_info.layerSync = true;
Erik Språng59021ba2018-10-03 11:05:16 +0200344 layers_[0].state = TemporalLayer::State::kKeyFrame;
345 layers_[1].state = TemporalLayer::State::kKeyFrame;
346 active_layer_ = 1;
philipel9df33532019-03-04 16:37:50 +0100347 info->template_structure =
348 GetTemplateStructure(number_of_temporal_layers_);
Erik Språngdb6335e2019-04-01 16:22:57 +0200349 } else if (active_layer_ >= 0 && layers_[active_layer_].state ==
350 TemporalLayer::State::kKeyFrame) {
351 layers_[active_layer_].state = TemporalLayer::State::kNormal;
Erik Språng59021ba2018-10-03 11:05:16 +0200352 }
Elad Alon819661a2019-01-29 17:17:09 +0100353
philipel9df33532019-03-04 16:37:50 +0100354 vp8_info.useExplicitDependencies = true;
355 RTC_DCHECK_EQ(vp8_info.referencedBuffersCount, 0u);
356 RTC_DCHECK_EQ(vp8_info.updatedBuffersCount, 0u);
Elad Alon819661a2019-01-29 17:17:09 +0100357
358 // Note that |frame_config| is not derefernced if |is_keyframe|,
359 // meaning it's never dereferenced if the optional may be unset.
Niels Möller1f44bc12019-04-09 10:27:34 +0200360 for (int i = 0; i < static_cast<int>(Vp8FrameConfig::Buffer::kCount); ++i) {
philipelda5aa4d2019-04-26 13:37:37 +0200361 bool references = false;
362 bool updates = is_keyframe;
Niels Möller1f44bc12019-04-09 10:27:34 +0200363 if (!is_keyframe && dependency_info->frame_config.References(
364 static_cast<Vp8FrameConfig::Buffer>(i))) {
philipel9df33532019-03-04 16:37:50 +0100365 RTC_DCHECK_LT(vp8_info.referencedBuffersCount,
Elad Alon819661a2019-01-29 17:17:09 +0100366 arraysize(CodecSpecificInfoVP8::referencedBuffers));
philipelda5aa4d2019-04-26 13:37:37 +0200367 references = true;
philipel9df33532019-03-04 16:37:50 +0100368 vp8_info.referencedBuffers[vp8_info.referencedBuffersCount++] = i;
Elad Alon819661a2019-01-29 17:17:09 +0100369 }
370
Niels Möller1f44bc12019-04-09 10:27:34 +0200371 if (is_keyframe || dependency_info->frame_config.Updates(
372 static_cast<Vp8FrameConfig::Buffer>(i))) {
philipel9df33532019-03-04 16:37:50 +0100373 RTC_DCHECK_LT(vp8_info.updatedBuffersCount,
Elad Alon819661a2019-01-29 17:17:09 +0100374 arraysize(CodecSpecificInfoVP8::updatedBuffers));
philipelda5aa4d2019-04-26 13:37:37 +0200375 updates = true;
philipel9df33532019-03-04 16:37:50 +0100376 vp8_info.updatedBuffers[vp8_info.updatedBuffersCount++] = i;
Elad Alon819661a2019-01-29 17:17:09 +0100377 }
philipelda5aa4d2019-04-26 13:37:37 +0200378
379 if (references || updates)
380 generic_frame_info.encoder_buffers.emplace_back(i, references, updates);
Elad Alon819661a2019-01-29 17:17:09 +0100381 }
Erik Språng59021ba2018-10-03 11:05:16 +0200382 }
383
Sebastian Janssone64a6882019-03-01 18:04:07 +0100384 encode_framerate_.Update(1, rtc::TimeMillis());
Erik Språng59021ba2018-10-03 11:05:16 +0200385
386 if (number_of_temporal_layers_ == 1)
387 return;
388
389 RTC_DCHECK_NE(-1, active_layer_);
Erik Språng2c4c9142015-06-24 11:24:44 +0200390 if (layers_[active_layer_].state == TemporalLayer::State::kDropped) {
391 layers_[active_layer_].state = TemporalLayer::State::kQualityBoost;
392 }
393
394 if (qp != -1)
395 layers_[active_layer_].last_qp = qp;
396
397 if (active_layer_ == 0) {
Erik Språng59021ba2018-10-03 11:05:16 +0200398 layers_[0].debt_bytes_ += size_bytes;
399 layers_[1].debt_bytes_ += size_bytes;
sprangb0fdfea2016-03-01 05:51:16 -0800400 ++stats_.num_tl0_frames_;
401 stats_.tl0_target_bitrate_sum_ += layers_[0].target_rate_kbps_;
402 stats_.tl0_qp_sum_ += qp;
Erik Språng2c4c9142015-06-24 11:24:44 +0200403 } else if (active_layer_ == 1) {
Erik Språng59021ba2018-10-03 11:05:16 +0200404 layers_[1].debt_bytes_ += size_bytes;
sprangb0fdfea2016-03-01 05:51:16 -0800405 ++stats_.num_tl1_frames_;
406 stats_.tl1_target_bitrate_sum_ += layers_[1].target_rate_kbps_;
407 stats_.tl1_qp_sum_ += qp;
Erik Språng2c4c9142015-06-24 11:24:44 +0200408 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000409}
410
Elad Alon6796ec22019-04-15 10:07:50 +0200411void ScreenshareLayers::OnFrameDropped(size_t stream_index,
412 uint32_t rtp_timestamp) {
413 layers_[active_layer_].state = TemporalLayer::State::kDropped;
414 ++stats_.num_overshoots_;
415}
416
Elad Aloncde8ab22019-03-20 11:56:20 +0100417void ScreenshareLayers::OnPacketLossRateUpdate(float packet_loss_rate) {}
418
419void ScreenshareLayers::OnRttUpdate(int64_t rtt_ms) {}
420
Elad Alon6c371ca2019-04-04 12:28:51 +0200421void ScreenshareLayers::OnLossNotification(
Elad Alon123ee9b2019-04-17 12:48:06 +0200422 const VideoEncoder::LossNotification& loss_notification) {}
Elad Alon6c371ca2019-04-04 12:28:51 +0200423
philipel9df33532019-03-04 16:37:50 +0100424TemplateStructure ScreenshareLayers::GetTemplateStructure(
425 int num_layers) const {
426 RTC_CHECK_LT(num_layers, 3);
427 RTC_CHECK_GT(num_layers, 0);
428
429 TemplateStructure template_structure;
philipel361855b2019-03-28 14:12:10 +0100430 template_structure.num_decode_targets = num_layers;
philipel9df33532019-03-04 16:37:50 +0100431
432 using Builder = GenericFrameInfo::Builder;
433 switch (num_layers) {
434 case 1: {
435 template_structure.templates = {
436 Builder().T(0).Dtis("S").Build(),
437 Builder().T(0).Dtis("S").Fdiffs({1}).Build(),
438 };
439 return template_structure;
440 }
441 case 2: {
442 template_structure.templates = {
443 Builder().T(0).Dtis("SS").Build(),
444 Builder().T(0).Dtis("SS").Fdiffs({1}).Build(),
445 Builder().T(1).Dtis("-S").Fdiffs({1}).Build(),
446 };
447 return template_structure;
448 }
449 default:
450 RTC_NOTREACHED();
451 // To make the compiler happy!
452 return template_structure;
453 }
454}
455
Erik Språng2c4c9142015-06-24 11:24:44 +0200456bool ScreenshareLayers::TimeToSync(int64_t timestamp) const {
sprang2ddb8bd2016-02-04 03:59:52 -0800457 RTC_DCHECK_EQ(1, active_layer_);
henrikg91d6ede2015-09-17 00:24:34 -0700458 RTC_DCHECK_NE(-1, layers_[0].last_qp);
Erik Språng2c4c9142015-06-24 11:24:44 +0200459 if (layers_[1].last_qp == -1) {
460 // First frame in TL1 should only depend on TL0 since there are no
461 // previous frames in TL1.
462 return true;
463 }
464
henrikg91d6ede2015-09-17 00:24:34 -0700465 RTC_DCHECK_NE(-1, last_sync_timestamp_);
Erik Språng2c4c9142015-06-24 11:24:44 +0200466 int64_t timestamp_diff = timestamp - last_sync_timestamp_;
467 if (timestamp_diff > kMaxTimeBetweenSyncs) {
468 // After a certain time, force a sync frame.
469 return true;
470 } else if (timestamp_diff < kMinTimeBetweenSyncs) {
471 // If too soon from previous sync frame, don't issue a new one.
472 return false;
473 }
474 // Issue a sync frame if difference in quality between TL0 and TL1 isn't too
475 // large.
476 if (layers_[0].last_qp - layers_[1].last_qp < kQpDeltaThresholdForSync)
477 return true;
478 return false;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000479}
480
sprangc7805db2016-11-25 08:09:43 -0800481uint32_t ScreenshareLayers::GetCodecTargetBitrateKbps() const {
482 uint32_t target_bitrate_kbps = layers_[0].target_rate_kbps_;
483
484 if (number_of_temporal_layers_ > 1) {
485 // Calculate a codec target bitrate. This may be higher than TL0, gaining
486 // quality at the expense of frame rate at TL0. Constraints:
487 // - TL0 frame rate no less than framerate / kMaxTL0FpsReduction.
488 // - Target rate * kAcceptableTargetOvershoot should not exceed TL1 rate.
489 target_bitrate_kbps =
490 std::min(layers_[0].target_rate_kbps_ * kMaxTL0FpsReduction,
491 layers_[1].target_rate_kbps_ / kAcceptableTargetOvershoot);
492 }
493
494 return std::max(layers_[0].target_rate_kbps_, target_bitrate_kbps);
495}
496
Elad Alonfb087812019-05-02 23:25:34 +0200497Vp8EncoderConfig ScreenshareLayers::UpdateConfiguration(size_t stream_index) {
Elad Aloncde8ab22019-03-20 11:56:20 +0100498 RTC_DCHECK_LT(stream_index, StreamCount());
Elad Alonfb087812019-05-02 23:25:34 +0200499 RTC_DCHECK(min_qp_.has_value());
500 RTC_DCHECK(max_qp_.has_value());
Elad Aloncde8ab22019-03-20 11:56:20 +0100501
Elad Alonfb087812019-05-02 23:25:34 +0200502 const uint32_t target_bitrate_kbps = GetCodecTargetBitrateKbps();
Erik Språng27a457d2018-01-12 11:00:21 +0100503
504 // TODO(sprang): We _really_ need to make an overhaul of this class. :(
505 // If we're dropping frames in order to meet a target framerate, adjust the
506 // bitrate assigned to the encoder so the total average bitrate is correct.
507 float encoder_config_bitrate_kbps = target_bitrate_kbps;
508 if (target_framerate_ && capture_framerate_ &&
509 *target_framerate_ < *capture_framerate_) {
510 encoder_config_bitrate_kbps *=
511 static_cast<float>(*capture_framerate_) / *target_framerate_;
512 }
513
514 if (bitrate_updated_ ||
Elad Alonfb087812019-05-02 23:25:34 +0200515 encoder_config_.rc_target_bitrate !=
516 absl::make_optional(encoder_config_bitrate_kbps)) {
517 encoder_config_.rc_target_bitrate = encoder_config_bitrate_kbps;
Erik Språng08127a92016-11-16 16:41:30 +0100518
519 // Don't reconfigure qp limits during quality boost frames.
520 if (active_layer_ == -1 ||
521 layers_[active_layer_].state != TemporalLayer::State::kQualityBoost) {
Elad Alonfb087812019-05-02 23:25:34 +0200522 const int min_qp = min_qp_.value();
523 const int max_qp = max_qp_.value();
524
Erik Språng08127a92016-11-16 16:41:30 +0100525 // After a dropped frame, a frame with max qp will be encoded and the
526 // quality will then ramp up from there. To boost the speed of recovery,
sprang916170a2017-05-23 07:47:55 -0700527 // encode the next frame with lower max qp, if there is sufficient
528 // bandwidth to do so without causing excessive delay.
529 // TL0 is the most important to improve since the errors in this layer
530 // will propagate to TL1.
Erik Språng08127a92016-11-16 16:41:30 +0100531 // Currently, reduce max qp by 20% for TL0 and 15% for TL1.
sprang916170a2017-05-23 07:47:55 -0700532 if (layers_[1].target_rate_kbps_ >= kMinBitrateKbpsForQpBoost) {
Elad Alonfb087812019-05-02 23:25:34 +0200533 layers_[0].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 80) / 100);
534 layers_[1].enhanced_max_qp = min_qp + (((max_qp - min_qp) * 85) / 100);
sprang916170a2017-05-23 07:47:55 -0700535 } else {
536 layers_[0].enhanced_max_qp = -1;
537 layers_[1].enhanced_max_qp = -1;
538 }
Erik Språng08127a92016-11-16 16:41:30 +0100539 }
540
sprang0ad0de62017-01-11 05:01:32 -0800541 if (capture_framerate_) {
sprangac4a90d2016-12-28 05:58:07 -0800542 int avg_frame_size =
sprang0ad0de62017-01-11 05:01:32 -0800543 (target_bitrate_kbps * 1000) / (8 * *capture_framerate_);
sprang916170a2017-05-23 07:47:55 -0700544 // Allow max debt to be the size of a single optimal frame.
545 // TODO(sprang): Determine if this needs to be adjusted by some factor.
546 // (Lower values may cause more frame drops, higher may lead to queuing
547 // delays.)
548 max_debt_bytes_ = avg_frame_size;
Erik Språng08127a92016-11-16 16:41:30 +0100549 }
550
551 bitrate_updated_ = false;
Erik Språng08127a92016-11-16 16:41:30 +0100552 }
553
554 // Don't try to update boosts state if not active yet.
555 if (active_layer_ == -1)
Elad Alonfb087812019-05-02 23:25:34 +0200556 return encoder_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100557
Elad Alonfb087812019-05-02 23:25:34 +0200558 if (number_of_temporal_layers_ <= 1)
559 return encoder_config_;
Erik Språng2c4c9142015-06-24 11:24:44 +0200560
561 // If layer is in the quality boost state (following a dropped frame), update
562 // the configuration with the adjusted (lower) qp and set the state back to
563 // normal.
Elad Alonfb087812019-05-02 23:25:34 +0200564 unsigned int adjusted_max_qp = max_qp_.value(); // Set the normal max qp.
Erik Språngf1414702018-09-12 15:04:08 +0200565 if (layers_[active_layer_].state == TemporalLayer::State::kQualityBoost) {
566 if (layers_[active_layer_].enhanced_max_qp != -1) {
567 // Bitrate is high enough for quality boost, update max qp.
568 adjusted_max_qp = layers_[active_layer_].enhanced_max_qp;
569 }
570 // Regardless of qp, reset the boost state for the next frame.
Erik Språng2c4c9142015-06-24 11:24:44 +0200571 layers_[active_layer_].state = TemporalLayer::State::kNormal;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000572 }
Elad Alonfb087812019-05-02 23:25:34 +0200573 encoder_config_.rc_max_quantizer = adjusted_max_qp;
Erik Språng2c4c9142015-06-24 11:24:44 +0200574
Elad Alonfb087812019-05-02 23:25:34 +0200575 return encoder_config_;
Erik Språng2c4c9142015-06-24 11:24:44 +0200576}
577
578void ScreenshareLayers::TemporalLayer::UpdateDebt(int64_t delta_ms) {
579 uint32_t debt_reduction_bytes = target_rate_kbps_ * delta_ms / 8;
580 if (debt_reduction_bytes >= debt_bytes_) {
581 debt_bytes_ = 0;
582 } else {
583 debt_bytes_ -= debt_reduction_bytes;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000584 }
585}
sprang@webrtc.org70f74f32014-12-17 10:57:10 +0000586
sprangb0fdfea2016-03-01 05:51:16 -0800587void ScreenshareLayers::UpdateHistograms() {
588 if (stats_.first_frame_time_ms_ == -1)
589 return;
590 int64_t duration_sec =
Sebastian Janssone64a6882019-03-01 18:04:07 +0100591 (rtc::TimeMillis() - stats_.first_frame_time_ms_ + 500) / 1000;
sprangb0fdfea2016-03-01 05:51:16 -0800592 if (duration_sec >= metrics::kMinRunTimeInSeconds) {
593 RTC_HISTOGRAM_COUNTS_10000(
594 "WebRTC.Video.Screenshare.Layer0.FrameRate",
595 (stats_.num_tl0_frames_ + (duration_sec / 2)) / duration_sec);
596 RTC_HISTOGRAM_COUNTS_10000(
597 "WebRTC.Video.Screenshare.Layer1.FrameRate",
598 (stats_.num_tl1_frames_ + (duration_sec / 2)) / duration_sec);
599 int total_frames = stats_.num_tl0_frames_ + stats_.num_tl1_frames_;
asapersson58d992e2016-03-29 02:15:06 -0700600 RTC_HISTOGRAM_COUNTS_10000(
601 "WebRTC.Video.Screenshare.FramesPerDrop",
Ilya Nikolaevskiybf352982017-10-02 10:08:25 +0200602 (stats_.num_dropped_frames_ == 0
603 ? 0
604 : total_frames / stats_.num_dropped_frames_));
asapersson58d992e2016-03-29 02:15:06 -0700605 RTC_HISTOGRAM_COUNTS_10000(
606 "WebRTC.Video.Screenshare.FramesPerOvershoot",
607 (stats_.num_overshoots_ == 0 ? 0
608 : total_frames / stats_.num_overshoots_));
sprangb0fdfea2016-03-01 05:51:16 -0800609 if (stats_.num_tl0_frames_ > 0) {
610 RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer0.Qp",
611 stats_.tl0_qp_sum_ / stats_.num_tl0_frames_);
612 RTC_HISTOGRAM_COUNTS_10000(
613 "WebRTC.Video.Screenshare.Layer0.TargetBitrate",
614 stats_.tl0_target_bitrate_sum_ / stats_.num_tl0_frames_);
615 }
616 if (stats_.num_tl1_frames_ > 0) {
617 RTC_HISTOGRAM_COUNTS_10000("WebRTC.Video.Screenshare.Layer1.Qp",
618 stats_.tl1_qp_sum_ / stats_.num_tl1_frames_);
619 RTC_HISTOGRAM_COUNTS_10000(
620 "WebRTC.Video.Screenshare.Layer1.TargetBitrate",
621 stats_.tl1_target_bitrate_sum_ / stats_.num_tl1_frames_);
622 }
623 }
624}
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000625} // namespace webrtc