blob: 0018403889d75ebceaa67e9c8e1a4b818ad138b1 [file] [log] [blame]
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
Mirko Bonadei95adedb2018-11-19 09:52:37 +010012#ifdef RTC_ENABLE_VP9
13
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020014#include "modules/video_coding/codecs/vp9/vp9_impl.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000015
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020016#include <algorithm>
Sergey Silkind45b3452018-05-31 16:00:24 +020017#include <limits>
marpan@webrtc.org5b883172014-11-01 06:10:48 +000018#include <vector>
19
marpan@webrtc.org5b883172014-11-01 06:10:48 +000020#include "vpx/vp8cx.h"
21#include "vpx/vp8dx.h"
Yves Gerey665174f2018-06-19 15:03:05 +020022#include "vpx/vpx_decoder.h"
23#include "vpx/vpx_encoder.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000024
Karl Wiberg918f50c2018-07-05 11:40:33 +020025#include "absl/memory/memory.h"
Emircan Uysaler800787f2018-07-16 10:01:49 -070026#include "api/video/color_space.h"
Emircan Uysaler0823eec2018-07-13 17:10:00 -070027#include "api/video/i010_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "common_video/include/video_frame_buffer.h"
29#include "common_video/libyuv/include/webrtc_libyuv.h"
Sergey Silkind902d582018-05-18 17:31:19 +020030#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Sergey Silkin86684962018-03-28 19:32:37 +020031#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/checks.h"
Erik Språng4b4266f2019-01-23 12:48:13 +010033#include "rtc_base/experiments/rate_control_settings.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/keep_ref_until_done.h"
35#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020037#include "rtc_base/trace_event.h"
Sergey Silkin390f3582018-10-01 17:13:50 +020038#include "system_wrappers/include/field_trial.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000039
40namespace webrtc {
41
Sergey Silkind902d582018-05-18 17:31:19 +020042namespace {
Sergey Silkin390f3582018-10-01 17:13:50 +020043// Maps from gof_idx to encoder internal reference frame buffer index. These
44// maps work for 1,2 and 3 temporal layers with GOF length of 1,2 and 4 frames.
45uint8_t kRefBufIdx[4] = {0, 0, 0, 1};
46uint8_t kUpdBufIdx[4] = {0, 0, 1, 0};
47
Sergey Silkinb674cd12018-10-09 10:59:06 +020048int kMaxNumTiles4kVideo = 8;
49
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +010050// Maximum allowed PID difference for differnet per-layer frame-rate case.
Ilya Nikolaevskiy3a656d12019-02-14 14:44:22 +010051const int kMaxAllowedPidDIff = 30;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +010052
Marco6e89b252015-07-07 14:40:38 -070053// Only positive speeds, range for real-time coding currently is: 5 - 8.
54// Lower means slower/better quality, higher means fastest/lower quality.
55int GetCpuSpeed(int width, int height) {
Alex Glaznevfecb7c32016-03-31 14:23:27 -070056#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
Marco002f0d02015-12-17 09:49:31 -080057 return 8;
58#else
Marco6e89b252015-07-07 14:40:38 -070059 // For smaller resolutions, use lower speed setting (get some coding gain at
60 // the cost of increased encoding complexity).
61 if (width * height <= 352 * 288)
62 return 5;
63 else
64 return 7;
Marco002f0d02015-12-17 09:49:31 -080065#endif
Marco6e89b252015-07-07 14:40:38 -070066}
Emircan Uysaler715cc232018-07-23 17:50:43 -070067// Helper class for extracting VP9 colorspace.
68ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t,
69 vpx_color_range_t range_t,
70 unsigned int bit_depth) {
Johannes Kronb47ccc32018-12-07 11:12:44 +010071 ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kUnspecified;
72 ColorSpace::TransferID transfer = ColorSpace::TransferID::kUnspecified;
73 ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kUnspecified;
Emircan Uysaler715cc232018-07-23 17:50:43 -070074 switch (space_t) {
75 case VPX_CS_BT_601:
76 case VPX_CS_SMPTE_170:
77 primaries = ColorSpace::PrimaryID::kSMPTE170M;
78 transfer = ColorSpace::TransferID::kSMPTE170M;
79 matrix = ColorSpace::MatrixID::kSMPTE170M;
80 break;
81 case VPX_CS_SMPTE_240:
82 primaries = ColorSpace::PrimaryID::kSMPTE240M;
83 transfer = ColorSpace::TransferID::kSMPTE240M;
84 matrix = ColorSpace::MatrixID::kSMPTE240M;
85 break;
86 case VPX_CS_BT_709:
87 primaries = ColorSpace::PrimaryID::kBT709;
88 transfer = ColorSpace::TransferID::kBT709;
89 matrix = ColorSpace::MatrixID::kBT709;
90 break;
91 case VPX_CS_BT_2020:
92 primaries = ColorSpace::PrimaryID::kBT2020;
93 switch (bit_depth) {
94 case 8:
95 transfer = ColorSpace::TransferID::kBT709;
96 break;
97 case 10:
98 transfer = ColorSpace::TransferID::kBT2020_10;
99 break;
100 default:
101 RTC_NOTREACHED();
102 break;
103 }
104 matrix = ColorSpace::MatrixID::kBT2020_NCL;
105 break;
106 case VPX_CS_SRGB:
107 primaries = ColorSpace::PrimaryID::kBT709;
108 transfer = ColorSpace::TransferID::kIEC61966_2_1;
109 matrix = ColorSpace::MatrixID::kBT709;
110 break;
111 default:
112 break;
113 }
114
115 ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
116 switch (range_t) {
117 case VPX_CR_STUDIO_RANGE:
118 range = ColorSpace::RangeID::kLimited;
119 break;
120 case VPX_CR_FULL_RANGE:
121 range = ColorSpace::RangeID::kFull;
122 break;
123 default:
124 break;
125 }
126 return ColorSpace(primaries, transfer, matrix, range);
127}
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100128
129bool MoreLayersEnabled(const VideoBitrateAllocation& first,
130 const VideoBitrateAllocation& second) {
131 for (size_t sl_idx = 0; sl_idx < kMaxSpatialLayers; ++sl_idx) {
132 if (first.GetSpatialLayerSum(sl_idx) > 0 &&
133 second.GetSpatialLayerSum(sl_idx) == 0) {
134 return true;
135 }
136 }
137 return false;
138}
139
Emircan Uysaler715cc232018-07-23 17:50:43 -0700140} // namespace
Marco6e89b252015-07-07 14:40:38 -0700141
asaperssona9455ab2015-07-31 06:10:09 -0700142void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
143 void* user_data) {
philipelcce46fc2015-12-21 03:04:49 -0800144 VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data);
asaperssona9455ab2015-07-31 06:10:09 -0700145 enc->GetEncodedLayerFrame(pkt);
146}
147
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700148VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000149 : encoded_image_(),
sprang3958ed82017-08-17 08:12:10 -0700150 encoded_complete_callback_(nullptr),
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700151 profile_(
152 ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000153 inited_(false),
154 timestamp_(0),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000155 cpu_speed_(3),
156 rc_max_intra_target_(0),
sprang3958ed82017-08-17 08:12:10 -0700157 encoder_(nullptr),
158 config_(nullptr),
159 raw_(nullptr),
160 input_image_(nullptr),
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200161 force_key_frame_(true),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200162 pics_since_key_(0),
asaperssona9455ab2015-07-31 06:10:09 -0700163 num_temporal_layers_(0),
philipelcfc319b2015-11-10 07:17:23 -0800164 num_spatial_layers_(0),
Sergey Silkin390f3582018-10-01 17:13:50 +0200165 num_active_spatial_layers_(0),
Erik Språngd3438aa2018-11-08 16:56:43 +0100166 layer_deactivation_requires_key_frame_(
167 field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")),
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200168 is_svc_(false),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200169 inter_layer_pred_(InterLayerPredMode::kOn),
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100170 external_ref_control_(false), // Set in InitEncode because of tests.
Erik Språng4b4266f2019-01-23 12:48:13 +0100171 trusted_rate_controller_(RateControlSettings::ParseFromFieldTrials()
172 .LibvpxVp9TrustedRateController()),
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100173 full_superframe_drop_(true),
174 first_frame_in_picture_(true),
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100175 ss_info_needed_(false),
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +0100176 is_flexible_mode_(false),
177 variable_framerate_experiment_(ParseVariableFramerateConfig(
178 "WebRTC-VP9VariableFramerateScreenshare")),
179 variable_framerate_controller_(
180 variable_framerate_experiment_.framerate_limit),
181 num_steady_state_frames_(0) {
philipeld1d03592019-03-01 13:53:55 +0100182 codec_ = {};
johannkoenig8225c402017-01-26 13:23:44 -0800183 memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000184}
185
186VP9EncoderImpl::~VP9EncoderImpl() {
187 Release();
188}
189
190int VP9EncoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100191 int ret_val = WEBRTC_VIDEO_CODEC_OK;
192
Niels Möllerdd1cc982019-02-07 18:52:50 +0100193 encoded_image_.Allocate(0);
sprang3958ed82017-08-17 08:12:10 -0700194 if (encoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +0100195 if (inited_) {
196 if (vpx_codec_destroy(encoder_)) {
197 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
198 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000199 }
200 delete encoder_;
sprang3958ed82017-08-17 08:12:10 -0700201 encoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000202 }
sprang3958ed82017-08-17 08:12:10 -0700203 if (config_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000204 delete config_;
sprang3958ed82017-08-17 08:12:10 -0700205 config_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000206 }
sprang3958ed82017-08-17 08:12:10 -0700207 if (raw_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000208 vpx_img_free(raw_);
sprang3958ed82017-08-17 08:12:10 -0700209 raw_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000210 }
211 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100212 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000213}
214
sprangce4aef12015-11-02 07:23:20 -0800215bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const {
216 // We check target_bitrate_bps of the 0th layer to see if the spatial layers
217 // (i.e. bitrates) were explicitly configured.
Sergey Silkinbeba1b22018-09-11 11:40:25 +0200218 return codec_.spatialLayers[0].targetBitrate > 0;
sprangce4aef12015-11-02 07:23:20 -0800219}
220
Erik Språng566124a2018-04-23 12:32:22 +0200221bool VP9EncoderImpl::SetSvcRates(
222 const VideoBitrateAllocation& bitrate_allocation) {
Sergey Silkin86684962018-03-28 19:32:37 +0200223 config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
Sergey Silkin86684962018-03-28 19:32:37 +0200224
sprangce4aef12015-11-02 07:23:20 -0800225 if (ExplicitlyConfiguredSpatialLayers()) {
Sergey Silkine7ce8882018-10-03 18:04:57 +0200226 const bool layer_activation_requires_key_frame =
227 inter_layer_pred_ == InterLayerPredMode::kOff ||
228 inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
229
Sergey Silkin86684962018-03-28 19:32:37 +0200230 for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
Sergey Silkine7ce8882018-10-03 18:04:57 +0200231 const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0);
Sergey Silkin86684962018-03-28 19:32:37 +0200232 config_->ss_target_bitrate[sl_idx] =
233 bitrate_allocation.GetSpatialLayerSum(sl_idx) / 1000;
234
235 for (size_t tl_idx = 0; tl_idx < num_temporal_layers_; ++tl_idx) {
236 config_->layer_target_bitrate[sl_idx * num_temporal_layers_ + tl_idx] =
237 bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
238 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200239
Sergey Silkine7ce8882018-10-03 18:04:57 +0200240 const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0);
241 if (!was_layer_active && is_active_layer &&
242 layer_activation_requires_key_frame) {
243 force_key_frame_ = true;
244 } else if (was_layer_active && !is_active_layer &&
245 layer_deactivation_requires_key_frame_) {
246 force_key_frame_ = true;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200247 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200248
Sergey Silkine7ce8882018-10-03 18:04:57 +0200249 if (!was_layer_active) {
Sergey Silkin96f2c972018-09-05 21:07:17 +0200250 // Reset frame rate controller if layer is resumed after pause.
251 framerate_controller_[sl_idx].Reset();
252 }
253
254 framerate_controller_[sl_idx].SetTargetRate(
Sergey Silkinbeba1b22018-09-11 11:40:25 +0200255 std::min(static_cast<float>(codec_.maxFramerate),
256 codec_.spatialLayers[sl_idx].maxFramerate));
sprangce4aef12015-11-02 07:23:20 -0800257 }
258 } else {
259 float rate_ratio[VPX_MAX_LAYERS] = {0};
260 float total = 0;
Sergey Silkin908689d2018-08-20 09:28:45 +0200261 for (int i = 0; i < num_spatial_layers_; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800262 if (svc_params_.scaling_factor_num[i] <= 0 ||
263 svc_params_.scaling_factor_den[i] <= 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100264 RTC_LOG(LS_ERROR) << "Scaling factors not specified!";
sprangce4aef12015-11-02 07:23:20 -0800265 return false;
266 }
Yves Gerey665174f2018-06-19 15:03:05 +0200267 rate_ratio[i] = static_cast<float>(svc_params_.scaling_factor_num[i]) /
268 svc_params_.scaling_factor_den[i];
sprangce4aef12015-11-02 07:23:20 -0800269 total += rate_ratio[i];
270 }
271
Sergey Silkin908689d2018-08-20 09:28:45 +0200272 for (int i = 0; i < num_spatial_layers_; ++i) {
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200273 RTC_CHECK_GT(total, 0);
sprangce4aef12015-11-02 07:23:20 -0800274 config_->ss_target_bitrate[i] = static_cast<unsigned int>(
275 config_->rc_target_bitrate * rate_ratio[i] / total);
276 if (num_temporal_layers_ == 1) {
277 config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i];
278 } else if (num_temporal_layers_ == 2) {
279 config_->layer_target_bitrate[i * num_temporal_layers_] =
280 config_->ss_target_bitrate[i] * 2 / 3;
281 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
282 config_->ss_target_bitrate[i];
283 } else if (num_temporal_layers_ == 3) {
284 config_->layer_target_bitrate[i * num_temporal_layers_] =
285 config_->ss_target_bitrate[i] / 2;
286 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
287 config_->layer_target_bitrate[i * num_temporal_layers_] +
288 (config_->ss_target_bitrate[i] / 4);
289 config_->layer_target_bitrate[i * num_temporal_layers_ + 2] =
290 config_->ss_target_bitrate[i];
291 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100292 RTC_LOG(LS_ERROR) << "Unsupported number of temporal layers: "
293 << num_temporal_layers_;
sprangce4aef12015-11-02 07:23:20 -0800294 return false;
295 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200296
297 framerate_controller_[i].SetTargetRate(codec_.maxFramerate);
asaperssona9455ab2015-07-31 06:10:09 -0700298 }
299 }
Sergey Silkin908689d2018-08-20 09:28:45 +0200300
301 num_active_spatial_layers_ = 0;
302 for (int i = 0; i < num_spatial_layers_; ++i) {
303 if (config_->ss_target_bitrate[i] > 0) {
304 ++num_active_spatial_layers_;
305 }
306 }
307 RTC_DCHECK_GT(num_active_spatial_layers_, 0);
308
asaperssona9455ab2015-07-31 06:10:09 -0700309 return true;
310}
311
Erik Språng16cb8f52019-04-12 13:59:09 +0200312void VP9EncoderImpl::SetRates(const RateControlParameters& parameters) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000313 if (!inited_) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200314 RTC_LOG(LS_WARNING) << "SetRates() calll while uninitialzied.";
315 return;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000316 }
317 if (encoder_->err) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200318 RTC_LOG(LS_WARNING) << "Encoder in error state: " << encoder_->err;
319 return;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000320 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200321 if (parameters.framerate_fps < 1.0) {
322 RTC_LOG(LS_WARNING) << "Unsupported framerate: "
323 << parameters.framerate_fps;
324 return;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000325 }
326 // Update bit rate
Erik Språng08127a92016-11-16 16:41:30 +0100327 if (codec_.maxBitrate > 0 &&
Erik Språng16cb8f52019-04-12 13:59:09 +0200328 parameters.bitrate.get_sum_kbps() > codec_.maxBitrate) {
329 RTC_LOG(LS_WARNING) << "Target bitrate exceeds maximum: "
330 << parameters.bitrate.get_sum_kbps() << " vs "
331 << codec_.maxBitrate;
332 return;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000333 }
Erik Språng08127a92016-11-16 16:41:30 +0100334
Erik Språng16cb8f52019-04-12 13:59:09 +0200335 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
336 requested_bitrate_allocation_ = parameters.bitrate;
asaperssona9455ab2015-07-31 06:10:09 -0700337
Erik Språng16cb8f52019-04-12 13:59:09 +0200338 return;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000339}
340
341int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
342 int number_of_cores,
pkasting@chromium.org4591fbd2014-11-20 22:28:14 +0000343 size_t /*max_payload_size*/) {
sprang3958ed82017-08-17 08:12:10 -0700344 if (inst == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000345 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
346 }
347 if (inst->maxFramerate < 1) {
348 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
349 }
350 // Allow zero to represent an unspecified maxBitRate
351 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
352 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
353 }
354 if (inst->width < 1 || inst->height < 1) {
355 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
356 }
357 if (number_of_cores < 1) {
358 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
359 }
hta257dc392016-10-25 09:05:06 -0700360 if (inst->VP9().numberOfTemporalLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700361 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
362 }
ilnik2a8c2f52017-02-15 02:23:28 -0800363 // libvpx probably does not support more than 3 spatial layers.
364 if (inst->VP9().numberOfSpatialLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700365 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
366 }
philipelcfc319b2015-11-10 07:17:23 -0800367
asapersson86956de2016-01-26 01:05:20 -0800368 int ret_val = Release();
369 if (ret_val < 0) {
370 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000371 }
sprang3958ed82017-08-17 08:12:10 -0700372 if (encoder_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000373 encoder_ = new vpx_codec_ctx_t;
374 }
sprang3958ed82017-08-17 08:12:10 -0700375 if (config_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000376 config_ = new vpx_codec_enc_cfg_t;
377 }
378 timestamp_ = 0;
379 if (&codec_ != inst) {
380 codec_ = *inst;
381 }
asaperssona9455ab2015-07-31 06:10:09 -0700382
Sergey Silkinb0bd03b2018-09-28 14:56:39 +0200383 force_key_frame_ = true;
384 pics_since_key_ = 0;
385
hta257dc392016-10-25 09:05:06 -0700386 num_spatial_layers_ = inst->VP9().numberOfSpatialLayers;
Niels Möller65fb4042018-04-25 14:46:06 +0200387 RTC_DCHECK_GT(num_spatial_layers_, 0);
hta257dc392016-10-25 09:05:06 -0700388 num_temporal_layers_ = inst->VP9().numberOfTemporalLayers;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200389 if (num_temporal_layers_ == 0) {
asaperssona9455ab2015-07-31 06:10:09 -0700390 num_temporal_layers_ = 1;
Sergey Silkin042661b2018-09-10 10:39:35 +0000391 }
Sergey Silkinae9e1882018-09-05 21:07:17 +0200392
Sergey Silkin96f2c972018-09-05 21:07:17 +0200393 framerate_controller_ = std::vector<FramerateController>(
394 num_spatial_layers_, FramerateController(codec_.maxFramerate));
395
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200396 is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200397
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000398 // Allocate memory for encoded image
Niels Möller48a79462018-12-07 16:21:18 +0100399 size_t frame_capacity =
nisseeb44b392017-04-28 07:18:05 -0700400 CalcBufferSize(VideoType::kI420, codec_.width, codec_.height);
Niels Möllerdd1cc982019-02-07 18:52:50 +0100401 encoded_image_.Allocate(frame_capacity);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000402 encoded_image_._completeFrame = true;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000403 // Populate encoder configuration with default values.
404 if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) {
405 return WEBRTC_VIDEO_CODEC_ERROR;
406 }
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700407
408 vpx_img_fmt img_fmt = VPX_IMG_FMT_NONE;
409 unsigned int bits_for_storage = 8;
410 switch (profile_) {
411 case VP9Profile::kProfile0:
412 img_fmt = VPX_IMG_FMT_I420;
413 bits_for_storage = 8;
414 config_->g_bit_depth = VPX_BITS_8;
415 config_->g_profile = 0;
416 config_->g_input_bit_depth = 8;
417 break;
418 case VP9Profile::kProfile2:
419 img_fmt = VPX_IMG_FMT_I42016;
420 bits_for_storage = 16;
421 config_->g_bit_depth = VPX_BITS_10;
422 config_->g_profile = 2;
423 config_->g_input_bit_depth = 10;
424 break;
425 }
426
427 // Creating a wrapper to the image - setting image data to nullptr. Actual
428 // pointer will be set in encode. Setting align to 1, as it is meaningless
429 // (actual memory is not allocated).
430 raw_ =
431 vpx_img_wrap(nullptr, img_fmt, codec_.width, codec_.height, 1, nullptr);
432 raw_->bit_depth = bits_for_storage;
433
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000434 config_->g_w = codec_.width;
435 config_->g_h = codec_.height;
436 config_->rc_target_bitrate = inst->startBitrate; // in kbit/s
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200437 config_->g_error_resilient = is_svc_ ? VPX_ERROR_RESILIENT_DEFAULT : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000438 // Setting the time base of the codec.
439 config_->g_timebase.num = 1;
440 config_->g_timebase.den = 90000;
441 config_->g_lag_in_frames = 0; // 0- no frame lagging
442 config_->g_threads = 1;
443 // Rate control settings.
hta257dc392016-10-25 09:05:06 -0700444 config_->rc_dropframe_thresh = inst->VP9().frameDroppingOn ? 30 : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000445 config_->rc_end_usage = VPX_CBR;
446 config_->g_pass = VPX_RC_ONE_PASS;
Ilya Nikolaevskiy3a656d12019-02-14 14:44:22 +0100447 config_->rc_min_quantizer =
448 codec_.mode == VideoCodecMode::kScreensharing ? 8 : 2;
marpan@webrtc.orgdc8a9da2015-01-27 23:08:24 +0000449 config_->rc_max_quantizer = 52;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000450 config_->rc_undershoot_pct = 50;
451 config_->rc_overshoot_pct = 50;
452 config_->rc_buf_initial_sz = 500;
453 config_->rc_buf_optimal_sz = 600;
454 config_->rc_buf_sz = 1000;
455 // Set the maximum target size of any key-frame.
456 rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz);
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +0100457 // Key-frame interval is enforced manually by this wrapper.
458 config_->kf_mode = VPX_KF_DISABLED;
459 // TODO(webm:1592): work-around for libvpx issue, as it can still
460 // put some key-frames at will even in VPX_KF_DISABLED kf_mode.
461 config_->kf_max_dist = inst->VP9().keyFrameInterval;
462 config_->kf_min_dist = config_->kf_max_dist;
hta257dc392016-10-25 09:05:06 -0700463 config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000464 // Determine number of threads based on the image size and #cores.
philipelcce46fc2015-12-21 03:04:49 -0800465 config_->g_threads =
466 NumberOfThreads(config_->g_w, config_->g_h, number_of_cores);
asaperssona9455ab2015-07-31 06:10:09 -0700467
Marco6e89b252015-07-07 14:40:38 -0700468 cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h);
asaperssona9455ab2015-07-31 06:10:09 -0700469
hta257dc392016-10-25 09:05:06 -0700470 is_flexible_mode_ = inst->VP9().flexibleMode;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200471
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100472 inter_layer_pred_ = inst->VP9().interLayerPred;
473
474 different_framerates_used_ = false;
475 for (size_t sl_idx = 1; sl_idx < num_spatial_layers_; ++sl_idx) {
476 if (std::abs(codec_.spatialLayers[sl_idx].maxFramerate -
477 codec_.spatialLayers[0].maxFramerate) > 1e-9) {
478 different_framerates_used_ = true;
479 }
480 }
481
482 if (different_framerates_used_ && !is_flexible_mode_) {
483 RTC_LOG(LS_ERROR) << "Flexible mode required for different framerates on "
484 "different spatial layers";
485 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
486 }
487
488 // External reference control is required for different frame rate on spatial
489 // layers because libvpx generates rtp incompatible references in this case.
490 external_ref_control_ = field_trial::IsEnabled("WebRTC-Vp9ExternalRefCtrl") ||
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +0100491 different_framerates_used_ ||
492 inter_layer_pred_ == InterLayerPredMode::kOn;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100493
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200494 if (num_temporal_layers_ == 1) {
asaperssona9455ab2015-07-31 06:10:09 -0700495 gof_.SetGofInfoVP9(kTemporalStructureMode1);
496 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING;
497 config_->ts_number_layers = 1;
498 config_->ts_rate_decimator[0] = 1;
499 config_->ts_periodicity = 1;
500 config_->ts_layer_id[0] = 0;
501 } else if (num_temporal_layers_ == 2) {
502 gof_.SetGofInfoVP9(kTemporalStructureMode2);
503 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
504 config_->ts_number_layers = 2;
505 config_->ts_rate_decimator[0] = 2;
506 config_->ts_rate_decimator[1] = 1;
507 config_->ts_periodicity = 2;
508 config_->ts_layer_id[0] = 0;
509 config_->ts_layer_id[1] = 1;
510 } else if (num_temporal_layers_ == 3) {
511 gof_.SetGofInfoVP9(kTemporalStructureMode3);
512 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212;
513 config_->ts_number_layers = 3;
514 config_->ts_rate_decimator[0] = 4;
515 config_->ts_rate_decimator[1] = 2;
516 config_->ts_rate_decimator[2] = 1;
517 config_->ts_periodicity = 4;
518 config_->ts_layer_id[0] = 0;
519 config_->ts_layer_id[1] = 2;
520 config_->ts_layer_id[2] = 1;
521 config_->ts_layer_id[3] = 2;
522 } else {
523 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
524 }
525
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100526 if (external_ref_control_) {
527 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS;
528 if (num_temporal_layers_ > 1 && different_framerates_used_) {
529 // External reference control for several temporal layers with different
530 // frame rates on spatial layers is not implemented yet.
531 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
532 }
533 }
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200534 ref_buf_.clear();
535
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000536 return InitAndSetControlSettings(inst);
537}
538
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000539int VP9EncoderImpl::NumberOfThreads(int width,
540 int height,
541 int number_of_cores) {
542 // Keep the number of encoder threads equal to the possible number of column
543 // tiles, which is (1, 2, 4, 8). See comments below for VP9E_SET_TILE_COLUMNS.
544 if (width * height >= 1280 * 720 && number_of_cores > 4) {
545 return 4;
jianj23173a32017-07-12 16:11:09 -0700546 } else if (width * height >= 640 * 360 && number_of_cores > 2) {
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000547 return 2;
548 } else {
Yves Gerey665174f2018-06-19 15:03:05 +0200549// Use 2 threads for low res on ARM.
Jerome Jiang831af372017-12-05 10:44:35 -0800550#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
551 defined(WEBRTC_ANDROID)
552 if (width * height >= 320 * 180 && number_of_cores > 2) {
553 return 2;
554 }
555#endif
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000556 // 1 thread less than VGA.
557 return 1;
558 }
559}
560
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000561int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) {
Åsa Perssonff24c042015-12-04 10:58:08 +0100562 // Set QP-min/max per spatial and temporal layer.
563 int tot_num_layers = num_spatial_layers_ * num_temporal_layers_;
564 for (int i = 0; i < tot_num_layers; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800565 svc_params_.max_quantizers[i] = config_->rc_max_quantizer;
566 svc_params_.min_quantizers[i] = config_->rc_min_quantizer;
Åsa Perssonff24c042015-12-04 10:58:08 +0100567 }
asaperssona9455ab2015-07-31 06:10:09 -0700568 config_->ss_number_layers = num_spatial_layers_;
sprangce4aef12015-11-02 07:23:20 -0800569 if (ExplicitlyConfiguredSpatialLayers()) {
570 for (int i = 0; i < num_spatial_layers_; ++i) {
571 const auto& layer = codec_.spatialLayers[i];
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200572 RTC_CHECK_GT(layer.width, 0);
Sergey Silkin13e74342018-03-02 12:28:00 +0100573 const int scale_factor = codec_.width / layer.width;
574 RTC_DCHECK_GT(scale_factor, 0);
575
576 // Ensure scaler factor is integer.
577 if (scale_factor * layer.width != codec_.width) {
578 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
579 }
580
581 // Ensure scale factor is the same in both dimensions.
582 if (scale_factor * layer.height != codec_.height) {
583 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
584 }
585
586 // Ensure scale factor is power of two.
587 const bool is_pow_of_two = (scale_factor & (scale_factor - 1)) == 0;
588 if (!is_pow_of_two) {
589 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
590 }
591
592 svc_params_.scaling_factor_num[i] = 1;
593 svc_params_.scaling_factor_den[i] = scale_factor;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200594
595 RTC_DCHECK_GT(codec_.spatialLayers[i].maxFramerate, 0);
596 RTC_DCHECK_LE(codec_.spatialLayers[i].maxFramerate, codec_.maxFramerate);
597 if (i > 0) {
598 // Frame rate of high spatial layer is supposed to be equal or higher
599 // than frame rate of low spatial layer.
600 RTC_DCHECK_GE(codec_.spatialLayers[i].maxFramerate,
601 codec_.spatialLayers[i - 1].maxFramerate);
602 }
sprangce4aef12015-11-02 07:23:20 -0800603 }
604 } else {
605 int scaling_factor_num = 256;
606 for (int i = num_spatial_layers_ - 1; i >= 0; --i) {
sprangce4aef12015-11-02 07:23:20 -0800607 // 1:2 scaling in each dimension.
johannkoenig8225c402017-01-26 13:23:44 -0800608 svc_params_.scaling_factor_num[i] = scaling_factor_num;
609 svc_params_.scaling_factor_den[i] = 256;
sprangce4aef12015-11-02 07:23:20 -0800610 }
asaperssona9455ab2015-07-31 06:10:09 -0700611 }
612
Sergey Silkin86684962018-03-28 19:32:37 +0200613 SvcRateAllocator init_allocator(codec_);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100614 current_bitrate_allocation_ = init_allocator.GetAllocation(
Sergey Silkin86684962018-03-28 19:32:37 +0200615 inst->startBitrate * 1000, inst->maxFramerate);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100616 if (!SetSvcRates(current_bitrate_allocation_)) {
asaperssona9455ab2015-07-31 06:10:09 -0700617 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
618 }
619
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700620 const vpx_codec_err_t rv = vpx_codec_enc_init(
621 encoder_, vpx_codec_vp9_cx(), config_,
622 config_->g_bit_depth == VPX_BITS_8 ? 0 : VPX_CODEC_USE_HIGHBITDEPTH);
623 if (rv != VPX_CODEC_OK) {
624 RTC_LOG(LS_ERROR) << "Init error: " << vpx_codec_err_to_string(rv);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000625 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
626 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000627 vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_);
628 vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT,
629 rc_max_intra_target_);
630 vpx_codec_control(encoder_, VP9E_SET_AQ_MODE,
hta257dc392016-10-25 09:05:06 -0700631 inst->VP9().adaptiveQpMode ? 3 : 0);
asaperssona9455ab2015-07-31 06:10:09 -0700632
jianj822e5932017-07-12 16:09:58 -0700633 vpx_codec_control(encoder_, VP9E_SET_FRAME_PARALLEL_DECODING, 0);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100634 vpx_codec_control(encoder_, VP9E_SET_SVC_GF_TEMPORAL_REF, 0);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200635
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200636 if (is_svc_) {
637 vpx_codec_control(encoder_, VP9E_SET_SVC, 1);
638 vpx_codec_control(encoder_, VP9E_SET_SVC_PARAMETERS, &svc_params_);
asaperssona9455ab2015-07-31 06:10:09 -0700639 }
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200640
641 if (num_spatial_layers_ > 1) {
642 switch (inter_layer_pred_) {
643 case InterLayerPredMode::kOn:
644 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 0);
645 break;
646 case InterLayerPredMode::kOff:
647 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 1);
648 break;
649 case InterLayerPredMode::kOnKeyPic:
650 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 2);
651 break;
652 default:
653 RTC_NOTREACHED();
654 }
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200655
Sergey Silkinee203362018-05-30 11:34:08 +0200656 // Configure encoder to drop entire superframe whenever it needs to drop
657 // a layer. This mode is prefered over per-layer dropping which causes
658 // quality flickering and is not compatible with RTP non-flexible mode.
659 vpx_svc_frame_drop_t svc_drop_frame;
660 memset(&svc_drop_frame, 0, sizeof(svc_drop_frame));
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100661 svc_drop_frame.framedrop_mode =
662 full_superframe_drop_ ? FULL_SUPERFRAME_DROP : CONSTRAINED_LAYER_DROP;
Sergey Silkind45b3452018-05-31 16:00:24 +0200663 svc_drop_frame.max_consec_drop = std::numeric_limits<int>::max();
Sergey Silkinee203362018-05-30 11:34:08 +0200664 for (size_t i = 0; i < num_spatial_layers_; ++i) {
665 svc_drop_frame.framedrop_thresh[i] = config_->rc_dropframe_thresh;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200666 }
Sergey Silkinee203362018-05-30 11:34:08 +0200667 vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200668 }
669
asaperssona9455ab2015-07-31 06:10:09 -0700670 // Register callback for getting each spatial layer.
671 vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = {
philipelcce46fc2015-12-21 03:04:49 -0800672 VP9EncoderImpl::EncoderOutputCodedPacketCallback,
673 reinterpret_cast<void*>(this)};
674 vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK,
675 reinterpret_cast<void*>(&cbp));
asaperssona9455ab2015-07-31 06:10:09 -0700676
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000677 // Control function to set the number of column tiles in encoding a frame, in
678 // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns.
679 // The number tile columns will be capped by the encoder based on image size
680 // (minimum width of tile column is 256 pixels, maximum is 4096).
681 vpx_codec_control(encoder_, VP9E_SET_TILE_COLUMNS, (config_->g_threads >> 1));
jianjcb5d1152017-03-28 23:56:08 -0700682
683 // Turn on row-based multithreading.
684 vpx_codec_control(encoder_, VP9E_SET_ROW_MT, 1);
jianj6bf57e32017-06-05 13:43:49 -0700685
Alex Glaznevfecb7c32016-03-31 14:23:27 -0700686#if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
Yves Gerey665174f2018-06-19 15:03:05 +0200687 !defined(ANDROID)
jianj6bf57e32017-06-05 13:43:49 -0700688 // Do not enable the denoiser on ARM since optimization is pending.
689 // Denoiser is on by default on other platforms.
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000690 vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY,
hta257dc392016-10-25 09:05:06 -0700691 inst->VP9().denoisingOn ? 1 : 0);
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000692#endif
jianj6bf57e32017-06-05 13:43:49 -0700693
Niels Möllere3cf3d02018-06-13 11:52:16 +0200694 if (codec_.mode == VideoCodecMode::kScreensharing) {
ivica242d6382015-09-04 06:13:23 -0700695 // Adjust internal parameters to screen content.
696 vpx_codec_control(encoder_, VP9E_SET_TUNE_CONTENT, 1);
ivica242d6382015-09-04 06:13:23 -0700697 }
Marco2520e722015-09-16 14:05:00 -0700698 // Enable encoder skip of static/low content blocks.
699 vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000700 inited_ = true;
701 return WEBRTC_VIDEO_CODEC_OK;
702}
703
704uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) {
705 // Set max to the optimal buffer level (normalized by target BR),
706 // and scaled by a scale_par.
707 // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps].
708 // This value is presented in percentage of perFrameBw:
709 // perFrameBw = targetBR[Kbps] * 1000 / framerate.
710 // The target in % is as follows:
711 float scale_par = 0.5;
712 uint32_t target_pct =
713 optimal_buffer_size * scale_par * codec_.maxFramerate / 10;
714 // Don't go below 3 times the per frame bandwidth.
715 const uint32_t min_intra_size = 300;
philipelcce46fc2015-12-21 03:04:49 -0800716 return (target_pct < min_intra_size) ? min_intra_size : target_pct;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000717}
718
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700719int VP9EncoderImpl::Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100720 const std::vector<VideoFrameType>* frame_types) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000721 if (!inited_) {
722 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
723 }
sprang3958ed82017-08-17 08:12:10 -0700724 if (encoded_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000725 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
726 }
Erik Språngbfa5d5d2019-02-04 16:40:19 +0100727 if (num_active_spatial_layers_ == 0) {
728 // All spatial layers are disabled, return without encoding anything.
729 return WEBRTC_VIDEO_CODEC_OK;
730 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200731
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000732 // We only support one stream at the moment.
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200733 if (frame_types && !frame_types->empty()) {
Niels Möller8f7ce222019-03-21 15:43:58 +0100734 if ((*frame_types)[0] == VideoFrameType::kVideoFrameKey) {
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200735 force_key_frame_ = true;
736 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000737 }
Sergey Silkind902d582018-05-18 17:31:19 +0200738
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +0100739 if (pics_since_key_ + 1 ==
740 static_cast<size_t>(codec_.VP9()->keyFrameInterval)) {
741 force_key_frame_ = true;
742 }
743
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100744 vpx_svc_layer_id_t layer_id = {0};
745 if (!force_key_frame_) {
Patrik Höglundd4d254f2018-11-29 15:09:31 +0000746 const size_t gof_idx = (pics_since_key_ + 1) % gof_.num_frames_in_gof;
747 layer_id.temporal_layer_id = gof_.temporal_idx[gof_idx];
Sergey Silkin96f2c972018-09-05 21:07:17 +0200748
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100749 if (VideoCodecMode::kScreensharing == codec_.mode) {
750 const uint32_t frame_timestamp_ms =
751 1000 * input_image.timestamp() / kVideoPayloadTypeFrequency;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200752
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +0100753 // To ensure that several rate-limiters with different limits don't
754 // interfere, they must be queried in order of increasing limit.
755
756 bool use_steady_state_limiter =
757 variable_framerate_experiment_.enabled &&
758 input_image.update_rect().IsEmpty() &&
759 num_steady_state_frames_ >=
760 variable_framerate_experiment_.frames_before_steady_state;
761
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100762 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +0100763 const float layer_fps =
764 framerate_controller_[layer_id.spatial_layer_id].GetTargetRate();
765 // Use steady state rate-limiter at the correct place.
766 if (use_steady_state_limiter &&
767 layer_fps > variable_framerate_experiment_.framerate_limit - 1e-9) {
768 if (variable_framerate_controller_.DropFrame(frame_timestamp_ms)) {
769 layer_id.spatial_layer_id = num_active_spatial_layers_;
770 }
771 // Break always: if rate limiter triggered frame drop, no need to
772 // continue; otherwise, the rate is less than the next limiters.
773 break;
774 }
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100775 if (framerate_controller_[sl_idx].DropFrame(frame_timestamp_ms)) {
776 ++layer_id.spatial_layer_id;
777 } else {
778 break;
779 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200780 }
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +0100781
782 if (use_steady_state_limiter &&
783 layer_id.spatial_layer_id < num_active_spatial_layers_) {
784 variable_framerate_controller_.AddFrame(frame_timestamp_ms);
785 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200786 }
Patrik Höglundd4d254f2018-11-29 15:09:31 +0000787
788 RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
789 if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
790 // Drop entire picture.
791 return WEBRTC_VIDEO_CODEC_OK;
792 }
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100793 }
Patrik Höglundd4d254f2018-11-29 15:09:31 +0000794
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100795 for (int sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
796 layer_id.temporal_layer_id_per_spatial[sl_idx] = layer_id.temporal_layer_id;
797 }
798
799 vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
800
801 if (requested_bitrate_allocation_) {
802 bool more_layers_requested = MoreLayersEnabled(
803 *requested_bitrate_allocation_, current_bitrate_allocation_);
804 bool less_layers_requested = MoreLayersEnabled(
805 current_bitrate_allocation_, *requested_bitrate_allocation_);
806 // In SVC can enable new layers only if all lower layers are encoded and at
807 // the base temporal layer.
808 // This will delay rate allocation change until the next frame on the base
809 // spatial layer.
810 // In KSVC or simulcast modes KF will be generated for a new layer, so can
811 // update allocation any time.
812 bool can_upswitch =
813 inter_layer_pred_ != InterLayerPredMode::kOn ||
814 (layer_id.spatial_layer_id == 0 && layer_id.temporal_layer_id == 0);
815 if (!more_layers_requested || can_upswitch) {
816 current_bitrate_allocation_ = *requested_bitrate_allocation_;
817 requested_bitrate_allocation_ = absl::nullopt;
818 if (!SetSvcRates(current_bitrate_allocation_)) {
819 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
820 }
821 if (less_layers_requested || more_layers_requested) {
822 ss_info_needed_ = true;
823 }
824 }
825 }
826
827 if (vpx_codec_enc_config_set(encoder_, config_)) {
828 return WEBRTC_VIDEO_CODEC_ERROR;
Sergey Silkind902d582018-05-18 17:31:19 +0200829 }
830
kwiberg352444f2016-11-28 15:58:53 -0800831 RTC_DCHECK_EQ(input_image.width(), raw_->d_w);
832 RTC_DCHECK_EQ(input_image.height(), raw_->d_h);
asaperssona9455ab2015-07-31 06:10:09 -0700833
834 // Set input image for use in the callback.
835 // This was necessary since you need some information from input_image.
836 // You can save only the necessary information (such as timestamp) instead of
837 // doing this.
838 input_image_ = &input_image;
839
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700840 // Keep reference to buffer until encode completes.
841 rtc::scoped_refptr<I420BufferInterface> i420_buffer;
842 rtc::scoped_refptr<I010BufferInterface> i010_buffer;
843 switch (profile_) {
844 case VP9Profile::kProfile0: {
845 i420_buffer = input_image.video_frame_buffer()->ToI420();
846 // Image in vpx_image_t format.
847 // Input image is const. VPX's raw image is not defined as const.
848 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
849 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU());
850 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV());
851 raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
852 raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU();
853 raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
854 break;
855 }
856 case VP9Profile::kProfile2: {
857 // We can inject kI010 frames directly for encode. All other formats
858 // should be converted to it.
859 switch (input_image.video_frame_buffer()->type()) {
860 case VideoFrameBuffer::Type::kI010: {
861 i010_buffer = input_image.video_frame_buffer()->GetI010();
862 break;
863 }
864 default: {
865 i010_buffer =
866 I010Buffer::Copy(*input_image.video_frame_buffer()->ToI420());
867 }
868 }
869 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(
870 reinterpret_cast<const uint8_t*>(i010_buffer->DataY()));
871 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(
872 reinterpret_cast<const uint8_t*>(i010_buffer->DataU()));
873 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(
874 reinterpret_cast<const uint8_t*>(i010_buffer->DataV()));
875 raw_->stride[VPX_PLANE_Y] = i010_buffer->StrideY() * 2;
876 raw_->stride[VPX_PLANE_U] = i010_buffer->StrideU() * 2;
877 raw_->stride[VPX_PLANE_V] = i010_buffer->StrideV() * 2;
878 break;
879 }
880 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000881
philipelcfc319b2015-11-10 07:17:23 -0800882 vpx_enc_frame_flags_t flags = 0;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200883 if (force_key_frame_) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000884 flags = VPX_EFLAG_FORCE_KF;
885 }
philipelcfc319b2015-11-10 07:17:23 -0800886
Sergey Silkin390f3582018-10-01 17:13:50 +0200887 if (external_ref_control_) {
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100888 vpx_svc_ref_frame_config_t ref_config =
889 SetReferences(force_key_frame_, layer_id.spatial_layer_id);
Sergey Silkina85995a2018-10-02 10:28:10 +0200890
891 if (VideoCodecMode::kScreensharing == codec_.mode) {
892 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
893 ref_config.duration[sl_idx] = static_cast<int64_t>(
894 90000 / framerate_controller_[sl_idx].GetTargetRate());
895 }
896 }
897
Sergey Silkin390f3582018-10-01 17:13:50 +0200898 vpx_codec_control(encoder_, VP9E_SET_SVC_REF_FRAME_CONFIG, &ref_config);
899 }
900
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100901 first_frame_in_picture_ = true;
902
Sergey Silkin96f2c972018-09-05 21:07:17 +0200903 // TODO(ssilkin): Frame duration should be specified per spatial layer
904 // since their frame rate can be different. For now calculate frame duration
905 // based on target frame rate of the highest spatial layer, which frame rate
906 // is supposed to be equal or higher than frame rate of low spatial layers.
907 // Also, timestamp should represent actual time passed since previous frame
908 // (not 'expected' time). Then rate controller can drain buffer more
909 // accurately.
910 RTC_DCHECK_GE(framerate_controller_.size(), num_active_spatial_layers_);
Sergey Silkinf87bb462018-09-17 09:37:37 +0200911 float target_framerate_fps =
912 (codec_.mode == VideoCodecMode::kScreensharing)
913 ? framerate_controller_[num_active_spatial_layers_ - 1]
914 .GetTargetRate()
915 : codec_.maxFramerate;
916 uint32_t duration = static_cast<uint32_t>(90000 / target_framerate_fps);
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700917 const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_,
918 duration, flags, VPX_DL_REALTIME);
919 if (rv != VPX_CODEC_OK) {
920 RTC_LOG(LS_ERROR) << "Encoding error: " << vpx_codec_err_to_string(rv)
921 << "\n"
922 << "Details: " << vpx_codec_error(encoder_) << "\n"
923 << vpx_codec_error_detail(encoder_);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000924 return WEBRTC_VIDEO_CODEC_ERROR;
925 }
926 timestamp_ += duration;
asaperssona9455ab2015-07-31 06:10:09 -0700927
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100928 if (!full_superframe_drop_) {
929 const bool end_of_picture = true;
930 DeliverBufferedFrame(end_of_picture);
931 }
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200932
asaperssona9455ab2015-07-31 06:10:09 -0700933 return WEBRTC_VIDEO_CODEC_OK;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000934}
935
936void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
Niels Möllerd3b8c632018-08-27 15:33:42 +0200937 absl::optional<int>* spatial_idx,
philipelcce46fc2015-12-21 03:04:49 -0800938 const vpx_codec_cx_pkt& pkt,
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100939 uint32_t timestamp) {
sprang3958ed82017-08-17 08:12:10 -0700940 RTC_CHECK(codec_specific != nullptr);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000941 codec_specific->codecType = kVideoCodecVP9;
philipelcce46fc2015-12-21 03:04:49 -0800942 CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9);
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200943
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100944 vp9_info->first_frame_in_picture = first_frame_in_picture_;
Sergey Silkin738b7e92018-08-23 16:37:27 +0200945 vp9_info->flexible_mode = is_flexible_mode_;
hta257dc392016-10-25 09:05:06 -0700946 vp9_info->ss_data_available =
Sergey Silkin738b7e92018-08-23 16:37:27 +0200947 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
asaperssona9455ab2015-07-31 06:10:09 -0700948
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100949 if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
950 pics_since_key_ = 0;
951 } else if (first_frame_in_picture_) {
952 ++pics_since_key_;
953 }
954
asaperssona9455ab2015-07-31 06:10:09 -0700955 vpx_svc_layer_id_t layer_id = {0};
956 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
957
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +0100958 // Can't have keyframe with non-zero temporal layer.
959 RTC_DCHECK(pics_since_key_ != 0 || layer_id.temporal_layer_id == 0);
960
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100961 if (ss_info_needed_ && layer_id.temporal_layer_id == 0 &&
962 layer_id.spatial_layer_id == 0) {
963 // Force SS info after the layers configuration has changed.
964 vp9_info->ss_data_available = true;
965 ss_info_needed_ = false;
966 }
967
sprang3958ed82017-08-17 08:12:10 -0700968 RTC_CHECK_GT(num_temporal_layers_, 0);
“Michael23c5a992018-06-21 11:07:21 -0500969 RTC_CHECK_GT(num_active_spatial_layers_, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700970 if (num_temporal_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700971 RTC_CHECK_EQ(layer_id.temporal_layer_id, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700972 vp9_info->temporal_idx = kNoTemporalIdx;
973 } else {
974 vp9_info->temporal_idx = layer_id.temporal_layer_id;
975 }
“Michael23c5a992018-06-21 11:07:21 -0500976 if (num_active_spatial_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700977 RTC_CHECK_EQ(layer_id.spatial_layer_id, 0);
Niels Möllerd3b8c632018-08-27 15:33:42 +0200978 *spatial_idx = absl::nullopt;
asaperssona9455ab2015-07-31 06:10:09 -0700979 } else {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200980 *spatial_idx = layer_id.spatial_layer_id;
asaperssona9455ab2015-07-31 06:10:09 -0700981 }
982 if (layer_id.spatial_layer_id != 0) {
983 vp9_info->ss_data_available = false;
984 }
985
asaperssona9455ab2015-07-31 06:10:09 -0700986 // TODO(asapersson): this info has to be obtained from the encoder.
asaperssoncb50c962015-11-18 01:58:55 -0800987 vp9_info->temporal_up_switch = false;
asaperssona9455ab2015-07-31 06:10:09 -0700988
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200989 const bool is_key_pic = (pics_since_key_ == 0);
990 const bool is_inter_layer_pred_allowed =
991 (inter_layer_pred_ == InterLayerPredMode::kOn ||
992 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic));
993
994 // Always set inter_layer_predicted to true on high layer frame if inter-layer
995 // prediction (ILP) is allowed even if encoder didn't actually use it.
996 // Setting inter_layer_predicted to false would allow receiver to decode high
997 // layer frame without decoding low layer frame. If that would happen (e.g.
998 // if low layer frame is lost) then receiver won't be able to decode next high
999 // layer frame which uses ILP.
1000 vp9_info->inter_layer_predicted =
Sergey Silkin88ce4ef2018-11-23 13:59:24 +01001001 first_frame_in_picture_ ? false : is_inter_layer_pred_allowed;
Sergey Silkin6a8f30e2018-04-26 11:03:49 +02001002
Sergey Silkinc5f152d2018-09-26 13:28:50 +02001003 // Mark all low spatial layer frames as references (not just frames of
1004 // active low spatial layers) if inter-layer prediction is enabled since
1005 // these frames are indirect references of high spatial layer, which can
1006 // later be enabled without key frame.
Sergey Silkin6a8f30e2018-04-26 11:03:49 +02001007 vp9_info->non_ref_for_inter_layer_pred =
Sergey Silkinc5f152d2018-09-26 13:28:50 +02001008 !is_inter_layer_pred_allowed ||
1009 layer_id.spatial_layer_id + 1 == num_spatial_layers_;
asapersson00ac85e2015-11-11 05:30:48 -08001010
ivica7f6a6fc2015-09-08 02:40:29 -07001011 // Always populate this, so that the packetizer can properly set the marker
1012 // bit.
“Michael23c5a992018-06-21 11:07:21 -05001013 vp9_info->num_spatial_layers = num_active_spatial_layers_;
philipelcfc319b2015-11-10 07:17:23 -08001014
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001015 vp9_info->num_ref_pics = 0;
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +01001016 FillReferenceIndices(pkt, pics_since_key_, vp9_info->inter_layer_predicted,
1017 vp9_info);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001018 if (vp9_info->flexible_mode) {
1019 vp9_info->gof_idx = kNoGofIdx;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001020 } else {
1021 vp9_info->gof_idx =
1022 static_cast<uint8_t>(pics_since_key_ % gof_.num_frames_in_gof);
1023 vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx];
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +01001024 RTC_DCHECK(vp9_info->num_ref_pics == gof_.num_ref_pics[vp9_info->gof_idx] ||
1025 vp9_info->num_ref_pics == 0);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001026 }
1027
1028 vp9_info->inter_pic_predicted = (!is_key_pic && vp9_info->num_ref_pics > 0);
1029
asaperssona9455ab2015-07-31 06:10:09 -07001030 if (vp9_info->ss_data_available) {
asaperssona9455ab2015-07-31 06:10:09 -07001031 vp9_info->spatial_layer_resolution_present = true;
“Michael23c5a992018-06-21 11:07:21 -05001032 for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
Yves Gerey665174f2018-06-19 15:03:05 +02001033 vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -08001034 svc_params_.scaling_factor_den[i];
Yves Gerey665174f2018-06-19 15:03:05 +02001035 vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -08001036 svc_params_.scaling_factor_den[i];
asaperssona9455ab2015-07-31 06:10:09 -07001037 }
Sergey Silkin738b7e92018-08-23 16:37:27 +02001038 if (vp9_info->flexible_mode) {
1039 vp9_info->gof.num_frames_in_gof = 0;
1040 } else {
asaperssona9455ab2015-07-31 06:10:09 -07001041 vp9_info->gof.CopyGofInfoVP9(gof_);
1042 }
1043 }
Sergey Silkin88ce4ef2018-11-23 13:59:24 +01001044
1045 first_frame_in_picture_ = false;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001046}
1047
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001048void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
1049 const size_t pic_num,
1050 const bool inter_layer_predicted,
1051 CodecSpecificInfoVP9* vp9_info) {
1052 vpx_svc_layer_id_t layer_id = {0};
1053 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1054
Sergey Silkin76be2952018-08-21 12:30:33 +02001055 const bool is_key_frame =
1056 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001057
1058 std::vector<RefFrameBuffer> ref_buf_list;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001059
Sergey Silkin76be2952018-08-21 12:30:33 +02001060 if (is_svc_) {
1061 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
1062 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
Sergey Silkin38fabff2019-03-21 10:08:40 +01001063 int ref_buf_flags = 0;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001064
Sergey Silkin76be2952018-08-21 12:30:33 +02001065 if (enc_layer_conf.reference_last[layer_id.spatial_layer_id]) {
1066 const size_t fb_idx =
1067 enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id];
1068 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +02001069 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
1070 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
1071 ref_buf_list.push_back(ref_buf_.at(fb_idx));
Sergey Silkin38fabff2019-03-21 10:08:40 +01001072 ref_buf_flags |= 1 << fb_idx;
Sergey Silkin82276592018-08-27 14:54:20 +02001073 }
Sergey Silkin76be2952018-08-21 12:30:33 +02001074 }
1075
1076 if (enc_layer_conf.reference_alt_ref[layer_id.spatial_layer_id]) {
1077 const size_t fb_idx =
1078 enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id];
1079 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +02001080 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
1081 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
1082 ref_buf_list.push_back(ref_buf_.at(fb_idx));
Sergey Silkin38fabff2019-03-21 10:08:40 +01001083 ref_buf_flags |= 1 << fb_idx;
Sergey Silkin82276592018-08-27 14:54:20 +02001084 }
Sergey Silkin76be2952018-08-21 12:30:33 +02001085 }
1086
1087 if (enc_layer_conf.reference_golden[layer_id.spatial_layer_id]) {
1088 const size_t fb_idx =
1089 enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id];
1090 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +02001091 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
1092 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
1093 ref_buf_list.push_back(ref_buf_.at(fb_idx));
Sergey Silkin38fabff2019-03-21 10:08:40 +01001094 ref_buf_flags |= 1 << fb_idx;
Sergey Silkin82276592018-08-27 14:54:20 +02001095 }
Sergey Silkin76be2952018-08-21 12:30:33 +02001096 }
Sergey Silkin38fabff2019-03-21 10:08:40 +01001097
1098 RTC_LOG(LS_VERBOSE) << "Frame " << pic_num << " sl "
1099 << layer_id.spatial_layer_id << " tl "
1100 << layer_id.temporal_layer_id << " refered buffers "
1101 << (ref_buf_flags & (1 << 0) ? 1 : 0)
1102 << (ref_buf_flags & (1 << 1) ? 1 : 0)
1103 << (ref_buf_flags & (1 << 2) ? 1 : 0)
1104 << (ref_buf_flags & (1 << 3) ? 1 : 0)
1105 << (ref_buf_flags & (1 << 4) ? 1 : 0)
1106 << (ref_buf_flags & (1 << 5) ? 1 : 0)
1107 << (ref_buf_flags & (1 << 6) ? 1 : 0)
1108 << (ref_buf_flags & (1 << 7) ? 1 : 0);
1109
Sergey Silkin76be2952018-08-21 12:30:33 +02001110 } else if (!is_key_frame) {
1111 RTC_DCHECK_EQ(num_spatial_layers_, 1);
1112 RTC_DCHECK_EQ(num_temporal_layers_, 1);
1113 // In non-SVC mode encoder doesn't provide reference list. Assume each frame
1114 // refers previous one, which is stored in buffer 0.
1115 ref_buf_list.push_back(ref_buf_.at(0));
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001116 }
1117
1118 size_t max_ref_temporal_layer_id = 0;
1119
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001120 std::vector<size_t> ref_pid_list;
1121
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001122 vp9_info->num_ref_pics = 0;
1123 for (const RefFrameBuffer& ref_buf : ref_buf_list) {
1124 RTC_DCHECK_LE(ref_buf.pic_num, pic_num);
1125 if (ref_buf.pic_num < pic_num) {
1126 if (inter_layer_pred_ != InterLayerPredMode::kOn) {
1127 // RTP spec limits temporal prediction to the same spatial layer.
1128 // It is safe to ignore this requirement if inter-layer prediction is
1129 // enabled for all frames when all base frames are relayed to receiver.
1130 RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id);
1131 }
1132 RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id);
1133
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001134 // Encoder may reference several spatial layers on the same previous
1135 // frame in case if some spatial layers are skipped on the current frame.
1136 // We shouldn't put duplicate references as it may break some old
1137 // clients and isn't RTP compatible.
1138 if (std::find(ref_pid_list.begin(), ref_pid_list.end(),
1139 ref_buf.pic_num) != ref_pid_list.end()) {
1140 continue;
1141 }
1142 ref_pid_list.push_back(ref_buf.pic_num);
1143
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001144 const size_t p_diff = pic_num - ref_buf.pic_num;
1145 RTC_DCHECK_LE(p_diff, 127UL);
1146
1147 vp9_info->p_diff[vp9_info->num_ref_pics] = static_cast<uint8_t>(p_diff);
1148 ++vp9_info->num_ref_pics;
1149
1150 max_ref_temporal_layer_id =
1151 std::max(max_ref_temporal_layer_id, ref_buf.temporal_layer_id);
1152 } else {
1153 RTC_DCHECK(inter_layer_predicted);
1154 // RTP spec only allows to use previous spatial layer for inter-layer
1155 // prediction.
1156 RTC_DCHECK_EQ(ref_buf.spatial_layer_id + 1, layer_id.spatial_layer_id);
1157 }
1158 }
1159
1160 vp9_info->temporal_up_switch =
1161 (max_ref_temporal_layer_id <
1162 static_cast<size_t>(layer_id.temporal_layer_id));
1163}
1164
1165void VP9EncoderImpl::UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
1166 const size_t pic_num) {
1167 vpx_svc_layer_id_t layer_id = {0};
1168 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1169
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001170 RefFrameBuffer frame_buf(pic_num, layer_id.spatial_layer_id,
1171 layer_id.temporal_layer_id);
1172
Sergey Silkin38fabff2019-03-21 10:08:40 +01001173 if (is_svc_) {
Sergey Silkin76be2952018-08-21 12:30:33 +02001174 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
1175 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
Sergey Silkin38fabff2019-03-21 10:08:40 +01001176 const int update_buffer_slot =
1177 enc_layer_conf.update_buffer_slot[layer_id.spatial_layer_id];
Sergey Silkin76be2952018-08-21 12:30:33 +02001178
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001179 for (size_t i = 0; i < kNumVp9Buffers; ++i) {
Sergey Silkin38fabff2019-03-21 10:08:40 +01001180 if (update_buffer_slot & (1 << i)) {
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001181 ref_buf_[i] = frame_buf;
1182 }
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001183 }
Sergey Silkin38fabff2019-03-21 10:08:40 +01001184
1185 RTC_LOG(LS_VERBOSE) << "Frame " << pic_num << " sl "
1186 << layer_id.spatial_layer_id << " tl "
1187 << layer_id.temporal_layer_id << " updated buffers "
1188 << (update_buffer_slot & (1 << 0) ? 1 : 0)
1189 << (update_buffer_slot & (1 << 1) ? 1 : 0)
1190 << (update_buffer_slot & (1 << 2) ? 1 : 0)
1191 << (update_buffer_slot & (1 << 3) ? 1 : 0)
1192 << (update_buffer_slot & (1 << 4) ? 1 : 0)
1193 << (update_buffer_slot & (1 << 5) ? 1 : 0)
1194 << (update_buffer_slot & (1 << 6) ? 1 : 0)
1195 << (update_buffer_slot & (1 << 7) ? 1 : 0);
Sergey Silkin76be2952018-08-21 12:30:33 +02001196 } else {
1197 RTC_DCHECK_EQ(num_spatial_layers_, 1);
1198 RTC_DCHECK_EQ(num_temporal_layers_, 1);
1199 // In non-svc mode encoder doesn't provide reference list. Assume each frame
1200 // is reference and stored in buffer 0.
1201 ref_buf_[0] = frame_buf;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001202 }
1203}
1204
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001205vpx_svc_ref_frame_config_t VP9EncoderImpl::SetReferences(
1206 bool is_key_pic,
1207 size_t first_active_spatial_layer_id) {
Sergey Silkin390f3582018-10-01 17:13:50 +02001208 // kRefBufIdx, kUpdBufIdx need to be updated to support longer GOFs.
1209 RTC_DCHECK_LE(gof_.num_frames_in_gof, 4);
1210
1211 vpx_svc_ref_frame_config_t ref_config;
1212 memset(&ref_config, 0, sizeof(ref_config));
1213
1214 const size_t num_temporal_refs = std::max(1, num_temporal_layers_ - 1);
1215 const bool is_inter_layer_pred_allowed =
1216 inter_layer_pred_ == InterLayerPredMode::kOn ||
1217 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic);
1218 absl::optional<int> last_updated_buf_idx;
1219
1220 // Put temporal reference to LAST and spatial reference to GOLDEN. Update
1221 // frame buffer (i.e. store encoded frame) if current frame is a temporal
1222 // reference (i.e. it belongs to a low temporal layer) or it is a spatial
1223 // reference. In later case, always store spatial reference in the last
1224 // reference frame buffer.
1225 // For the case of 3 temporal and 3 spatial layers we need 6 frame buffers
1226 // for temporal references plus 1 buffer for spatial reference. 7 buffers
1227 // in total.
1228
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001229 for (size_t sl_idx = first_active_spatial_layer_id;
1230 sl_idx < num_active_spatial_layers_; ++sl_idx) {
1231 const size_t curr_pic_num = is_key_pic ? 0 : pics_since_key_ + 1;
1232 const size_t gof_idx = curr_pic_num % gof_.num_frames_in_gof;
Sergey Silkin390f3582018-10-01 17:13:50 +02001233
1234 if (!is_key_pic) {
1235 // Set up temporal reference.
1236 const int buf_idx = sl_idx * num_temporal_refs + kRefBufIdx[gof_idx];
1237
1238 // Last reference frame buffer is reserved for spatial reference. It is
1239 // not supposed to be used for temporal prediction.
1240 RTC_DCHECK_LT(buf_idx, kNumVp9Buffers - 1);
1241
1242 // Sanity check that reference picture number is smaller than current
1243 // picture number.
Sergey Silkin390f3582018-10-01 17:13:50 +02001244 RTC_DCHECK_LT(ref_buf_[buf_idx].pic_num, curr_pic_num);
1245 const size_t pid_diff = curr_pic_num - ref_buf_[buf_idx].pic_num;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001246 // Incorrect spatial layer may be in the buffer due to a key-frame.
1247 const bool same_spatial_layer =
1248 ref_buf_[buf_idx].spatial_layer_id == sl_idx;
1249 bool correct_pid = false;
1250 if (different_framerates_used_) {
1251 correct_pid = pid_diff < kMaxAllowedPidDIff;
1252 } else {
1253 // Below code assumes single temporal referecence.
1254 RTC_DCHECK_EQ(gof_.num_ref_pics[gof_idx], 1);
1255 correct_pid = pid_diff == gof_.pid_diff[gof_idx][0];
1256 }
Sergey Silkin390f3582018-10-01 17:13:50 +02001257
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001258 if (same_spatial_layer && correct_pid) {
Sergey Silkin390f3582018-10-01 17:13:50 +02001259 ref_config.lst_fb_idx[sl_idx] = buf_idx;
1260 ref_config.reference_last[sl_idx] = 1;
1261 } else {
1262 // This reference doesn't match with one specified by GOF. This can
1263 // only happen if spatial layer is enabled dynamically without key
1264 // frame. Spatial prediction is supposed to be enabled in this case.
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001265 RTC_DCHECK(is_inter_layer_pred_allowed &&
1266 sl_idx > first_active_spatial_layer_id);
Sergey Silkin390f3582018-10-01 17:13:50 +02001267 }
1268 }
1269
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001270 if (is_inter_layer_pred_allowed && sl_idx > first_active_spatial_layer_id) {
Sergey Silkin390f3582018-10-01 17:13:50 +02001271 // Set up spatial reference.
1272 RTC_DCHECK(last_updated_buf_idx);
1273 ref_config.gld_fb_idx[sl_idx] = *last_updated_buf_idx;
1274 ref_config.reference_golden[sl_idx] = 1;
1275 } else {
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001276 RTC_DCHECK(ref_config.reference_last[sl_idx] != 0 ||
1277 sl_idx == first_active_spatial_layer_id ||
Sergey Silkin390f3582018-10-01 17:13:50 +02001278 inter_layer_pred_ == InterLayerPredMode::kOff);
1279 }
1280
1281 last_updated_buf_idx.reset();
1282
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +01001283 if (gof_.temporal_idx[gof_idx] < num_temporal_layers_ - 1 ||
1284 num_temporal_layers_ == 1) {
Sergey Silkin390f3582018-10-01 17:13:50 +02001285 last_updated_buf_idx = sl_idx * num_temporal_refs + kUpdBufIdx[gof_idx];
1286
1287 // Ensure last frame buffer is not used for temporal prediction (it is
1288 // reserved for spatial reference).
1289 RTC_DCHECK_LT(*last_updated_buf_idx, kNumVp9Buffers - 1);
1290 } else if (is_inter_layer_pred_allowed) {
1291 last_updated_buf_idx = kNumVp9Buffers - 1;
1292 }
1293
1294 if (last_updated_buf_idx) {
1295 ref_config.update_buffer_slot[sl_idx] = 1 << *last_updated_buf_idx;
1296 }
1297 }
1298
1299 return ref_config;
1300}
1301
asaperssona9455ab2015-07-31 06:10:09 -07001302int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) {
asapersson86956de2016-01-26 01:05:20 -08001303 RTC_DCHECK_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT);
asaperssona9455ab2015-07-31 06:10:09 -07001304
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001305 if (pkt->data.frame.sz == 0) {
1306 // Ignore dropped frame.
1307 return WEBRTC_VIDEO_CODEC_OK;
1308 }
1309
Sergey Silkin07f80cc2018-04-09 13:11:59 +02001310 vpx_svc_layer_id_t layer_id = {0};
1311 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1312
Sergey Silkin88ce4ef2018-11-23 13:59:24 +01001313 if (!full_superframe_drop_) {
1314 // Deliver buffered low spatial layer frame.
1315 const bool end_of_picture = false;
1316 DeliverBufferedFrame(end_of_picture);
1317 }
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001318
Niels Möller48a79462018-12-07 16:21:18 +01001319 if (pkt->data.frame.sz > encoded_image_.capacity()) {
Niels Möllerdd1cc982019-02-07 18:52:50 +01001320 encoded_image_.Allocate(pkt->data.frame.sz);
asaperssond9f641e2016-01-21 01:11:35 -08001321 }
Niels Möller24871e42019-01-17 11:31:13 +01001322 memcpy(encoded_image_.data(), pkt->data.frame.buf, pkt->data.frame.sz);
Niels Möller77536a22019-01-15 08:50:01 +01001323 encoded_image_.set_size(pkt->data.frame.sz);
asaperssond9f641e2016-01-21 01:11:35 -08001324
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001325 const bool is_key_frame =
1326 (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
1327 // Ensure encoder issued key frame on request.
1328 RTC_DCHECK(is_key_frame || !force_key_frame_);
1329
asaperssona9455ab2015-07-31 06:10:09 -07001330 // Check if encoded frame is a key frame.
Niels Möller8f7ce222019-03-21 15:43:58 +01001331 encoded_image_._frameType = VideoFrameType::kVideoFrameDelta;
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001332 if (is_key_frame) {
Niels Möller8f7ce222019-03-21 15:43:58 +01001333 encoded_image_._frameType = VideoFrameType::kVideoFrameKey;
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001334 force_key_frame_ = false;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001335 }
Niels Möller77536a22019-01-15 08:50:01 +01001336 RTC_DCHECK_LE(encoded_image_.size(), encoded_image_.capacity());
asapersson86956de2016-01-26 01:05:20 -08001337
philipeld1d03592019-03-01 13:53:55 +01001338 codec_specific_ = {};
Niels Möllerd3b8c632018-08-27 15:33:42 +02001339 absl::optional<int> spatial_index;
1340 PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt,
Sergey Silkin88ce4ef2018-11-23 13:59:24 +01001341 input_image_->timestamp());
Niels Möllerd3b8c632018-08-27 15:33:42 +02001342 encoded_image_.SetSpatialIndex(spatial_index);
asaperssona9455ab2015-07-31 06:10:09 -07001343
Ilya Nikolaevskiy2ec0c652019-01-18 11:56:48 +01001344 UpdateReferenceBuffers(*pkt, pics_since_key_);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001345
Niels Möller77536a22019-01-15 08:50:01 +01001346 TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_.size());
Niels Möller23775882018-08-16 10:24:12 +02001347 encoded_image_.SetTimestamp(input_image_->timestamp());
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001348 encoded_image_.capture_time_ms_ = input_image_->render_time_ms();
1349 encoded_image_.rotation_ = input_image_->rotation();
Niels Möllere3cf3d02018-06-13 11:52:16 +02001350 encoded_image_.content_type_ = (codec_.mode == VideoCodecMode::kScreensharing)
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001351 ? VideoContentType::SCREENSHARE
1352 : VideoContentType::UNSPECIFIED;
1353 encoded_image_._encodedHeight =
1354 pkt->data.frame.height[layer_id.spatial_layer_id];
1355 encoded_image_._encodedWidth =
1356 pkt->data.frame.width[layer_id.spatial_layer_id];
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +02001357 encoded_image_.timing_.flags = VideoSendTiming::kInvalid;
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001358 int qp = -1;
1359 vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp);
1360 encoded_image_.qp_ = qp;
Johannes Kron4749e4e2018-11-21 10:18:18 +01001361 encoded_image_.SetColorSpace(input_image_->color_space());
ilnik04f4d122017-06-19 07:18:55 -07001362
Sergey Silkin88ce4ef2018-11-23 13:59:24 +01001363 if (full_superframe_drop_) {
1364 const bool end_of_picture = encoded_image_.SpatialIndex().value_or(0) + 1 ==
1365 num_active_spatial_layers_;
1366 DeliverBufferedFrame(end_of_picture);
1367 }
1368
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001369 return WEBRTC_VIDEO_CODEC_OK;
1370}
1371
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001372void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) {
Niels Möller77536a22019-01-15 08:50:01 +01001373 if (encoded_image_.size() > 0) {
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001374 codec_specific_.codecSpecific.VP9.end_of_picture = end_of_picture;
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001375
1376 // No data partitioning in VP9, so 1 partition only.
1377 int part_idx = 0;
1378 RTPFragmentationHeader frag_info;
1379 frag_info.VerifyAndAllocateFragmentationHeader(1);
1380 frag_info.fragmentationOffset[part_idx] = 0;
Niels Möller77536a22019-01-15 08:50:01 +01001381 frag_info.fragmentationLength[part_idx] = encoded_image_.size();
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001382 frag_info.fragmentationPlType[part_idx] = 0;
1383 frag_info.fragmentationTimeDiff[part_idx] = 0;
1384
1385 encoded_complete_callback_->OnEncodedImage(encoded_image_, &codec_specific_,
1386 &frag_info);
Sergey Silkind902d582018-05-18 17:31:19 +02001387
Sergey Silkin96f2c972018-09-05 21:07:17 +02001388 if (codec_.mode == VideoCodecMode::kScreensharing) {
1389 const uint8_t spatial_idx = encoded_image_.SpatialIndex().value_or(0);
1390 const uint32_t frame_timestamp_ms =
Niels Möller23775882018-08-16 10:24:12 +02001391 1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency;
Sergey Silkin96f2c972018-09-05 21:07:17 +02001392 framerate_controller_[spatial_idx].AddFrame(frame_timestamp_ms);
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +01001393
1394 const size_t steady_state_size = SteadyStateSize(
1395 spatial_idx, codec_specific_.codecSpecific.VP9.temporal_idx);
1396
1397 // Only frames on spatial layers, which may be limited in a steady state
1398 // are considered for steady state detection.
1399 if (framerate_controller_[spatial_idx].GetTargetRate() >
1400 variable_framerate_experiment_.framerate_limit + 1e-9) {
1401 if (encoded_image_.qp_ <=
1402 variable_framerate_experiment_.steady_state_qp &&
1403 encoded_image_.size() <= steady_state_size) {
1404 ++num_steady_state_frames_;
1405 } else {
1406 num_steady_state_frames_ = 0;
1407 }
1408 }
Sergey Silkind902d582018-05-18 17:31:19 +02001409 }
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +01001410 encoded_image_.set_size(0);
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001411 }
1412}
1413
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001414int VP9EncoderImpl::RegisterEncodeCompleteCallback(
1415 EncodedImageCallback* callback) {
1416 encoded_complete_callback_ = callback;
1417 return WEBRTC_VIDEO_CODEC_OK;
1418}
1419
Erik Språng727d1642018-11-07 16:54:15 +01001420VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
1421 EncoderInfo info;
1422 info.supports_native_handle = false;
1423 info.implementation_name = "libvpx";
1424 info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
Erik Språngd3438aa2018-11-08 16:56:43 +01001425 info.has_trusted_rate_controller = trusted_rate_controller_;
Mirta Dvornicic897a9912018-11-30 13:12:21 +01001426 info.is_hardware_accelerated = false;
1427 info.has_internal_source = false;
Erik Språngdbdd8392019-01-17 15:27:50 +01001428 for (size_t si = 0; si < num_spatial_layers_; ++si) {
1429 info.fps_allocation[si].clear();
1430 if (!codec_.spatialLayers[si].active) {
1431 continue;
1432 }
1433 // This spatial layer may already use a fraction of the total frame rate.
1434 const float sl_fps_fraction =
1435 codec_.spatialLayers[si].maxFramerate / codec_.maxFramerate;
1436 for (size_t ti = 0; ti < num_temporal_layers_; ++ti) {
1437 const uint32_t decimator =
1438 num_temporal_layers_ <= 1 ? 1 : config_->ts_rate_decimator[ti];
1439 RTC_DCHECK_GT(decimator, 0);
1440 info.fps_allocation[si].push_back(rtc::saturated_cast<uint8_t>(
1441 EncoderInfo::kMaxFramerateFraction * (sl_fps_fraction / decimator)));
1442 }
1443 }
Erik Språng727d1642018-11-07 16:54:15 +01001444 return info;
Peter Boströmb7d9a972015-12-18 16:01:11 +01001445}
1446
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +01001447size_t VP9EncoderImpl::SteadyStateSize(int sid, int tid) {
1448 const size_t bitrate_bps = current_bitrate_allocation_.GetBitrate(
1449 sid, tid == kNoTemporalIdx ? 0 : tid);
1450 const float fps = (codec_.mode == VideoCodecMode::kScreensharing)
1451 ? framerate_controller_[sid].GetTargetRate()
1452 : codec_.maxFramerate;
1453 return static_cast<size_t>(
1454 bitrate_bps / (8 * fps) *
1455 (100 -
1456 variable_framerate_experiment_.steady_state_undershoot_percentage) /
1457 100 +
1458 0.5);
1459}
1460
1461// static
1462VP9EncoderImpl::VariableFramerateExperiment
1463VP9EncoderImpl::ParseVariableFramerateConfig(std::string group_name) {
1464 FieldTrialFlag enabled = FieldTrialFlag("Enabled");
1465 FieldTrialParameter<double> framerate_limit("min_fps", 5.0);
1466 FieldTrialParameter<int> qp("min_qp", 32);
1467 FieldTrialParameter<int> undershoot_percentage("undershoot", 30);
1468 FieldTrialParameter<int> frames_before_steady_state(
1469 "frames_before_steady_state", 5);
1470 ParseFieldTrial({&enabled, &framerate_limit, &qp, &undershoot_percentage,
1471 &frames_before_steady_state},
1472 field_trial::FindFullName(group_name));
1473 VariableFramerateExperiment config;
1474 config.enabled = enabled.Get();
1475 config.framerate_limit = framerate_limit.Get();
1476 config.steady_state_qp = qp.Get();
1477 config.steady_state_undershoot_percentage = undershoot_percentage.Get();
1478 config.frames_before_steady_state = frames_before_steady_state.Get();
1479
1480 return config;
1481}
1482
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001483VP9DecoderImpl::VP9DecoderImpl()
sprang3958ed82017-08-17 08:12:10 -07001484 : decode_complete_callback_(nullptr),
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001485 inited_(false),
sprang3958ed82017-08-17 08:12:10 -07001486 decoder_(nullptr),
philipel9d7d75b2018-04-04 13:03:01 +02001487 key_frame_required_(true) {}
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001488
1489VP9DecoderImpl::~VP9DecoderImpl() {
1490 inited_ = true; // in order to do the actual release
1491 Release();
Henrik Boström9695d852015-05-06 10:42:15 +02001492 int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse();
1493 if (num_buffers_in_use > 0) {
1494 // The frame buffers are reference counted and frames are exposed after
1495 // decoding. There may be valid usage cases where previous frames are still
1496 // referenced after ~VP9DecoderImpl that is not a leak.
Mirko Bonadei675513b2017-11-09 11:09:25 +01001497 RTC_LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still "
1498 << "referenced during ~VP9DecoderImpl.";
Henrik Boström9695d852015-05-06 10:42:15 +02001499 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001500}
1501
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001502int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001503 int ret_val = Release();
1504 if (ret_val < 0) {
1505 return ret_val;
1506 }
Sergey Silkinb674cd12018-10-09 10:59:06 +02001507
sprang3958ed82017-08-17 08:12:10 -07001508 if (decoder_ == nullptr) {
pbos@webrtc.orge728ee02014-12-17 13:43:55 +00001509 decoder_ = new vpx_codec_ctx_t;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001510 }
philipelcce46fc2015-12-21 03:04:49 -08001511 vpx_codec_dec_cfg_t cfg;
Sergey Silkinb674cd12018-10-09 10:59:06 +02001512 memset(&cfg, 0, sizeof(cfg));
1513
1514 // We want to use multithreading when decoding high resolution videos. But,
1515 // since we don't know resolution of input stream at this stage, we always
1516 // enable it.
1517 cfg.threads = std::min(number_of_cores, kMaxNumTiles4kVideo);
1518
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001519 vpx_codec_flags_t flags = 0;
1520 if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
1521 return WEBRTC_VIDEO_CODEC_MEMORY;
1522 }
Henrik Boström9695d852015-05-06 10:42:15 +02001523
1524 if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) {
1525 return WEBRTC_VIDEO_CODEC_MEMORY;
1526 }
1527
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001528 inited_ = true;
1529 // Always start with a complete key frame.
1530 key_frame_required_ = true;
1531 return WEBRTC_VIDEO_CODEC_OK;
1532}
1533
1534int VP9DecoderImpl::Decode(const EncodedImage& input_image,
1535 bool missing_frames,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001536 int64_t /*render_time_ms*/) {
1537 if (!inited_) {
1538 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1539 }
sprang3958ed82017-08-17 08:12:10 -07001540 if (decode_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001541 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1542 }
1543 // Always start with a complete key frame.
1544 if (key_frame_required_) {
Niels Möller8f7ce222019-03-21 15:43:58 +01001545 if (input_image._frameType != VideoFrameType::kVideoFrameKey)
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001546 return WEBRTC_VIDEO_CODEC_ERROR;
1547 // We have a key frame - is it complete?
1548 if (input_image._completeFrame) {
1549 key_frame_required_ = false;
1550 } else {
1551 return WEBRTC_VIDEO_CODEC_ERROR;
1552 }
1553 }
sprang3958ed82017-08-17 08:12:10 -07001554 vpx_codec_iter_t iter = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001555 vpx_image_t* img;
Niels Möller24871e42019-01-17 11:31:13 +01001556 const uint8_t* buffer = input_image.data();
Niels Möller77536a22019-01-15 08:50:01 +01001557 if (input_image.size() == 0) {
sprang3958ed82017-08-17 08:12:10 -07001558 buffer = nullptr; // Triggers full frame concealment.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001559 }
Henrik Boström9695d852015-05-06 10:42:15 +02001560 // During decode libvpx may get and release buffers from |frame_buffer_pool_|.
1561 // In practice libvpx keeps a few (~3-4) buffers alive at a time.
philipelcce46fc2015-12-21 03:04:49 -08001562 if (vpx_codec_decode(decoder_, buffer,
Niels Möller77536a22019-01-15 08:50:01 +01001563 static_cast<unsigned int>(input_image.size()), 0,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001564 VPX_DL_REALTIME)) {
1565 return WEBRTC_VIDEO_CODEC_ERROR;
1566 }
Henrik Boström9695d852015-05-06 10:42:15 +02001567 // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer.
1568 // It may be released by libvpx during future vpx_codec_decode or
1569 // vpx_codec_destroy calls.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001570 img = vpx_codec_get_frame(decoder_, &iter);
sakal7adadb12017-02-23 02:54:57 -08001571 int qp;
1572 vpx_codec_err_t vpx_ret =
1573 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
1574 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
Johannes Kron9973fa82018-11-07 14:39:26 +01001575 int ret = ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_,
Johannes Kron4749e4e2018-11-21 10:18:18 +01001576 qp, input_image.ColorSpace());
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001577 if (ret != 0) {
1578 return ret;
1579 }
1580 return WEBRTC_VIDEO_CODEC_OK;
1581}
1582
Danil Chapovalovb7698942019-02-05 11:32:19 +01001583int VP9DecoderImpl::ReturnFrame(
1584 const vpx_image_t* img,
1585 uint32_t timestamp,
1586 int64_t ntp_time_ms,
1587 int qp,
1588 const webrtc::ColorSpace* explicit_color_space) {
sprang3958ed82017-08-17 08:12:10 -07001589 if (img == nullptr) {
1590 // Decoder OK and nullptr image => No show frame.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001591 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1592 }
Henrik Boström9695d852015-05-06 10:42:15 +02001593
1594 // This buffer contains all of |img|'s image data, a reference counted
perkj14f41442015-11-30 22:15:45 -08001595 // Vp9FrameBuffer. (libvpx is done with the buffers after a few
Henrik Boström9695d852015-05-06 10:42:15 +02001596 // vpx_codec_decode calls or vpx_codec_destroy).
1597 Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer =
1598 static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv);
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001599
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -07001600 // The buffer can be used directly by the VideoFrame (without copy) by
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001601 // using a Wrapped*Buffer.
1602 rtc::scoped_refptr<VideoFrameBuffer> img_wrapped_buffer;
1603 switch (img->bit_depth) {
1604 case 8:
1605 img_wrapped_buffer = WrapI420Buffer(
philipelcce46fc2015-12-21 03:04:49 -08001606 img->d_w, img->d_h, img->planes[VPX_PLANE_Y],
1607 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U],
1608 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V],
1609 img->stride[VPX_PLANE_V],
Henrik Boström9695d852015-05-06 10:42:15 +02001610 // WrappedI420Buffer's mechanism for allowing the release of its frame
1611 // buffer is through a callback function. This is where we should
1612 // release |img_buffer|.
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001613 rtc::KeepRefUntilDone(img_buffer));
1614 break;
1615 case 10:
1616 img_wrapped_buffer = WrapI010Buffer(
1617 img->d_w, img->d_h,
1618 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]),
1619 img->stride[VPX_PLANE_Y] / 2,
1620 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]),
1621 img->stride[VPX_PLANE_U] / 2,
1622 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]),
1623 img->stride[VPX_PLANE_V] / 2, rtc::KeepRefUntilDone(img_buffer));
1624 break;
1625 default:
1626 RTC_NOTREACHED();
1627 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1628 }
nisseca6d5d12016-06-17 05:03:04 -07001629
Johannes Kron4749e4e2018-11-21 10:18:18 +01001630 auto builder = VideoFrame::Builder()
1631 .set_video_frame_buffer(img_wrapped_buffer)
1632 .set_timestamp_ms(0)
1633 .set_timestamp_rtp(timestamp)
1634 .set_ntp_time_ms(ntp_time_ms)
1635 .set_rotation(webrtc::kVideoRotation_0);
1636 if (explicit_color_space) {
Danil Chapovalovb7698942019-02-05 11:32:19 +01001637 builder.set_color_space(*explicit_color_space);
Johannes Kron4749e4e2018-11-21 10:18:18 +01001638 } else {
1639 builder.set_color_space(
1640 ExtractVP9ColorSpace(img->cs, img->range, img->bit_depth));
1641 }
1642
1643 VideoFrame decoded_image = builder.build();
1644
Danil Chapovalov0040b662018-06-18 10:48:16 +02001645 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001646 return WEBRTC_VIDEO_CODEC_OK;
1647}
1648
1649int VP9DecoderImpl::RegisterDecodeCompleteCallback(
1650 DecodedImageCallback* callback) {
1651 decode_complete_callback_ = callback;
1652 return WEBRTC_VIDEO_CODEC_OK;
1653}
1654
1655int VP9DecoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001656 int ret_val = WEBRTC_VIDEO_CODEC_OK;
1657
sprang3958ed82017-08-17 08:12:10 -07001658 if (decoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +01001659 if (inited_) {
1660 // When a codec is destroyed libvpx will release any buffers of
1661 // |frame_buffer_pool_| it is currently using.
1662 if (vpx_codec_destroy(decoder_)) {
1663 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
1664 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001665 }
1666 delete decoder_;
sprang3958ed82017-08-17 08:12:10 -07001667 decoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001668 }
Henrik Boström9695d852015-05-06 10:42:15 +02001669 // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
1670 // still referenced externally are deleted once fully released, not returning
1671 // to the pool.
1672 frame_buffer_pool_.ClearPool();
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001673 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001674 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001675}
Peter Boströmb7d9a972015-12-18 16:01:11 +01001676
1677const char* VP9DecoderImpl::ImplementationName() const {
1678 return "libvpx";
1679}
1680
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001681} // namespace webrtc
Mirko Bonadei95adedb2018-11-19 09:52:37 +01001682
1683#endif // RTC_ENABLE_VP9