blob: 427e446b3ae7d203e9eb74e834092f65aa26c0bd [file] [log] [blame]
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
Mirko Bonadei95adedb2018-11-19 09:52:37 +010012#ifdef RTC_ENABLE_VP9
13
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020014#include "modules/video_coding/codecs/vp9/vp9_impl.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000015
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020016#include <algorithm>
Sergey Silkind45b3452018-05-31 16:00:24 +020017#include <limits>
marpan@webrtc.org5b883172014-11-01 06:10:48 +000018#include <vector>
19
marpan@webrtc.org5b883172014-11-01 06:10:48 +000020#include "vpx/vp8cx.h"
21#include "vpx/vp8dx.h"
Yves Gerey665174f2018-06-19 15:03:05 +020022#include "vpx/vpx_decoder.h"
23#include "vpx/vpx_encoder.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000024
Karl Wiberg918f50c2018-07-05 11:40:33 +020025#include "absl/memory/memory.h"
Emircan Uysaler800787f2018-07-16 10:01:49 -070026#include "api/video/color_space.h"
Emircan Uysaler0823eec2018-07-13 17:10:00 -070027#include "api/video/i010_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "common_video/include/video_frame_buffer.h"
29#include "common_video/libyuv/include/webrtc_libyuv.h"
Sergey Silkind902d582018-05-18 17:31:19 +020030#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Sergey Silkin86684962018-03-28 19:32:37 +020031#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/checks.h"
33#include "rtc_base/keep_ref_until_done.h"
34#include "rtc_base/logging.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/timeutils.h"
36#include "rtc_base/trace_event.h"
Sergey Silkin390f3582018-10-01 17:13:50 +020037#include "system_wrappers/include/field_trial.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000038
39namespace webrtc {
40
Sergey Silkind902d582018-05-18 17:31:19 +020041namespace {
Erik Språngd3438aa2018-11-08 16:56:43 +010042const char kVp9TrustedRateControllerFieldTrial[] =
43 "WebRTC-LibvpxVp9TrustedRateController";
44
Sergey Silkin390f3582018-10-01 17:13:50 +020045// Maps from gof_idx to encoder internal reference frame buffer index. These
46// maps work for 1,2 and 3 temporal layers with GOF length of 1,2 and 4 frames.
47uint8_t kRefBufIdx[4] = {0, 0, 0, 1};
48uint8_t kUpdBufIdx[4] = {0, 0, 1, 0};
49
Sergey Silkinb674cd12018-10-09 10:59:06 +020050int kMaxNumTiles4kVideo = 8;
51
Marco6e89b252015-07-07 14:40:38 -070052// Only positive speeds, range for real-time coding currently is: 5 - 8.
53// Lower means slower/better quality, higher means fastest/lower quality.
54int GetCpuSpeed(int width, int height) {
Alex Glaznevfecb7c32016-03-31 14:23:27 -070055#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
Marco002f0d02015-12-17 09:49:31 -080056 return 8;
57#else
Marco6e89b252015-07-07 14:40:38 -070058 // For smaller resolutions, use lower speed setting (get some coding gain at
59 // the cost of increased encoding complexity).
60 if (width * height <= 352 * 288)
61 return 5;
62 else
63 return 7;
Marco002f0d02015-12-17 09:49:31 -080064#endif
Marco6e89b252015-07-07 14:40:38 -070065}
Emircan Uysaler715cc232018-07-23 17:50:43 -070066// Helper class for extracting VP9 colorspace.
67ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t,
68 vpx_color_range_t range_t,
69 unsigned int bit_depth) {
70 ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid;
71 ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid;
72 ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid;
73 switch (space_t) {
74 case VPX_CS_BT_601:
75 case VPX_CS_SMPTE_170:
76 primaries = ColorSpace::PrimaryID::kSMPTE170M;
77 transfer = ColorSpace::TransferID::kSMPTE170M;
78 matrix = ColorSpace::MatrixID::kSMPTE170M;
79 break;
80 case VPX_CS_SMPTE_240:
81 primaries = ColorSpace::PrimaryID::kSMPTE240M;
82 transfer = ColorSpace::TransferID::kSMPTE240M;
83 matrix = ColorSpace::MatrixID::kSMPTE240M;
84 break;
85 case VPX_CS_BT_709:
86 primaries = ColorSpace::PrimaryID::kBT709;
87 transfer = ColorSpace::TransferID::kBT709;
88 matrix = ColorSpace::MatrixID::kBT709;
89 break;
90 case VPX_CS_BT_2020:
91 primaries = ColorSpace::PrimaryID::kBT2020;
92 switch (bit_depth) {
93 case 8:
94 transfer = ColorSpace::TransferID::kBT709;
95 break;
96 case 10:
97 transfer = ColorSpace::TransferID::kBT2020_10;
98 break;
99 default:
100 RTC_NOTREACHED();
101 break;
102 }
103 matrix = ColorSpace::MatrixID::kBT2020_NCL;
104 break;
105 case VPX_CS_SRGB:
106 primaries = ColorSpace::PrimaryID::kBT709;
107 transfer = ColorSpace::TransferID::kIEC61966_2_1;
108 matrix = ColorSpace::MatrixID::kBT709;
109 break;
110 default:
111 break;
112 }
113
114 ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
115 switch (range_t) {
116 case VPX_CR_STUDIO_RANGE:
117 range = ColorSpace::RangeID::kLimited;
118 break;
119 case VPX_CR_FULL_RANGE:
120 range = ColorSpace::RangeID::kFull;
121 break;
122 default:
123 break;
124 }
125 return ColorSpace(primaries, transfer, matrix, range);
126}
127} // namespace
Marco6e89b252015-07-07 14:40:38 -0700128
asaperssona9455ab2015-07-31 06:10:09 -0700129void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
130 void* user_data) {
philipelcce46fc2015-12-21 03:04:49 -0800131 VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data);
asaperssona9455ab2015-07-31 06:10:09 -0700132 enc->GetEncodedLayerFrame(pkt);
133}
134
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700135VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000136 : encoded_image_(),
sprang3958ed82017-08-17 08:12:10 -0700137 encoded_complete_callback_(nullptr),
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700138 profile_(
139 ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000140 inited_(false),
141 timestamp_(0),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000142 cpu_speed_(3),
143 rc_max_intra_target_(0),
sprang3958ed82017-08-17 08:12:10 -0700144 encoder_(nullptr),
145 config_(nullptr),
146 raw_(nullptr),
147 input_image_(nullptr),
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200148 force_key_frame_(true),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200149 pics_since_key_(0),
asaperssona9455ab2015-07-31 06:10:09 -0700150 num_temporal_layers_(0),
philipelcfc319b2015-11-10 07:17:23 -0800151 num_spatial_layers_(0),
Sergey Silkin390f3582018-10-01 17:13:50 +0200152 num_active_spatial_layers_(0),
Erik Språngd3438aa2018-11-08 16:56:43 +0100153 layer_deactivation_requires_key_frame_(
154 field_trial::IsEnabled("WebRTC-Vp9IssueKeyFrameOnLayerDeactivation")),
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200155 is_svc_(false),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200156 inter_layer_pred_(InterLayerPredMode::kOn),
Sergey Silkin390f3582018-10-01 17:13:50 +0200157 external_ref_control_(
Erik Språngd3438aa2018-11-08 16:56:43 +0100158 field_trial::IsEnabled("WebRTC-Vp9ExternalRefCtrl")),
159 trusted_rate_controller_(
160 field_trial::IsEnabled(kVp9TrustedRateControllerFieldTrial)),
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200161 is_flexible_mode_(false) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000162 memset(&codec_, 0, sizeof(codec_));
johannkoenig8225c402017-01-26 13:23:44 -0800163 memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000164}
165
166VP9EncoderImpl::~VP9EncoderImpl() {
167 Release();
168}
169
170int VP9EncoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100171 int ret_val = WEBRTC_VIDEO_CODEC_OK;
172
sprang3958ed82017-08-17 08:12:10 -0700173 if (encoded_image_._buffer != nullptr) {
philipelcce46fc2015-12-21 03:04:49 -0800174 delete[] encoded_image_._buffer;
sprang3958ed82017-08-17 08:12:10 -0700175 encoded_image_._buffer = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000176 }
sprang3958ed82017-08-17 08:12:10 -0700177 if (encoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +0100178 if (inited_) {
179 if (vpx_codec_destroy(encoder_)) {
180 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
181 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000182 }
183 delete encoder_;
sprang3958ed82017-08-17 08:12:10 -0700184 encoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000185 }
sprang3958ed82017-08-17 08:12:10 -0700186 if (config_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000187 delete config_;
sprang3958ed82017-08-17 08:12:10 -0700188 config_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000189 }
sprang3958ed82017-08-17 08:12:10 -0700190 if (raw_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000191 vpx_img_free(raw_);
sprang3958ed82017-08-17 08:12:10 -0700192 raw_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000193 }
194 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100195 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000196}
197
sprangce4aef12015-11-02 07:23:20 -0800198bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const {
199 // We check target_bitrate_bps of the 0th layer to see if the spatial layers
200 // (i.e. bitrates) were explicitly configured.
Sergey Silkinbeba1b22018-09-11 11:40:25 +0200201 return codec_.spatialLayers[0].targetBitrate > 0;
sprangce4aef12015-11-02 07:23:20 -0800202}
203
Erik Språng566124a2018-04-23 12:32:22 +0200204bool VP9EncoderImpl::SetSvcRates(
205 const VideoBitrateAllocation& bitrate_allocation) {
Sergey Silkin86684962018-03-28 19:32:37 +0200206 config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
Sergey Silkin86684962018-03-28 19:32:37 +0200207
sprangce4aef12015-11-02 07:23:20 -0800208 if (ExplicitlyConfiguredSpatialLayers()) {
Sergey Silkine7ce8882018-10-03 18:04:57 +0200209 const bool layer_activation_requires_key_frame =
210 inter_layer_pred_ == InterLayerPredMode::kOff ||
211 inter_layer_pred_ == InterLayerPredMode::kOnKeyPic;
212
Sergey Silkin86684962018-03-28 19:32:37 +0200213 for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
Sergey Silkine7ce8882018-10-03 18:04:57 +0200214 const bool was_layer_active = (config_->ss_target_bitrate[sl_idx] > 0);
Sergey Silkin86684962018-03-28 19:32:37 +0200215 config_->ss_target_bitrate[sl_idx] =
216 bitrate_allocation.GetSpatialLayerSum(sl_idx) / 1000;
217
218 for (size_t tl_idx = 0; tl_idx < num_temporal_layers_; ++tl_idx) {
219 config_->layer_target_bitrate[sl_idx * num_temporal_layers_ + tl_idx] =
220 bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
221 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200222
Sergey Silkine7ce8882018-10-03 18:04:57 +0200223 const bool is_active_layer = (config_->ss_target_bitrate[sl_idx] > 0);
224 if (!was_layer_active && is_active_layer &&
225 layer_activation_requires_key_frame) {
226 force_key_frame_ = true;
227 } else if (was_layer_active && !is_active_layer &&
228 layer_deactivation_requires_key_frame_) {
229 force_key_frame_ = true;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200230 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200231
Sergey Silkine7ce8882018-10-03 18:04:57 +0200232 if (!was_layer_active) {
Sergey Silkin96f2c972018-09-05 21:07:17 +0200233 // Reset frame rate controller if layer is resumed after pause.
234 framerate_controller_[sl_idx].Reset();
235 }
236
237 framerate_controller_[sl_idx].SetTargetRate(
Sergey Silkinbeba1b22018-09-11 11:40:25 +0200238 std::min(static_cast<float>(codec_.maxFramerate),
239 codec_.spatialLayers[sl_idx].maxFramerate));
sprangce4aef12015-11-02 07:23:20 -0800240 }
241 } else {
242 float rate_ratio[VPX_MAX_LAYERS] = {0};
243 float total = 0;
Sergey Silkin908689d2018-08-20 09:28:45 +0200244 for (int i = 0; i < num_spatial_layers_; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800245 if (svc_params_.scaling_factor_num[i] <= 0 ||
246 svc_params_.scaling_factor_den[i] <= 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100247 RTC_LOG(LS_ERROR) << "Scaling factors not specified!";
sprangce4aef12015-11-02 07:23:20 -0800248 return false;
249 }
Yves Gerey665174f2018-06-19 15:03:05 +0200250 rate_ratio[i] = static_cast<float>(svc_params_.scaling_factor_num[i]) /
251 svc_params_.scaling_factor_den[i];
sprangce4aef12015-11-02 07:23:20 -0800252 total += rate_ratio[i];
253 }
254
Sergey Silkin908689d2018-08-20 09:28:45 +0200255 for (int i = 0; i < num_spatial_layers_; ++i) {
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200256 RTC_CHECK_GT(total, 0);
sprangce4aef12015-11-02 07:23:20 -0800257 config_->ss_target_bitrate[i] = static_cast<unsigned int>(
258 config_->rc_target_bitrate * rate_ratio[i] / total);
259 if (num_temporal_layers_ == 1) {
260 config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i];
261 } else if (num_temporal_layers_ == 2) {
262 config_->layer_target_bitrate[i * num_temporal_layers_] =
263 config_->ss_target_bitrate[i] * 2 / 3;
264 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
265 config_->ss_target_bitrate[i];
266 } else if (num_temporal_layers_ == 3) {
267 config_->layer_target_bitrate[i * num_temporal_layers_] =
268 config_->ss_target_bitrate[i] / 2;
269 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
270 config_->layer_target_bitrate[i * num_temporal_layers_] +
271 (config_->ss_target_bitrate[i] / 4);
272 config_->layer_target_bitrate[i * num_temporal_layers_ + 2] =
273 config_->ss_target_bitrate[i];
274 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100275 RTC_LOG(LS_ERROR) << "Unsupported number of temporal layers: "
276 << num_temporal_layers_;
sprangce4aef12015-11-02 07:23:20 -0800277 return false;
278 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200279
280 framerate_controller_[i].SetTargetRate(codec_.maxFramerate);
asaperssona9455ab2015-07-31 06:10:09 -0700281 }
282 }
Sergey Silkin908689d2018-08-20 09:28:45 +0200283
284 num_active_spatial_layers_ = 0;
285 for (int i = 0; i < num_spatial_layers_; ++i) {
286 if (config_->ss_target_bitrate[i] > 0) {
287 ++num_active_spatial_layers_;
288 }
289 }
290 RTC_DCHECK_GT(num_active_spatial_layers_, 0);
291
asaperssona9455ab2015-07-31 06:10:09 -0700292 return true;
293}
294
Erik Språng08127a92016-11-16 16:41:30 +0100295int VP9EncoderImpl::SetRateAllocation(
Erik Språng566124a2018-04-23 12:32:22 +0200296 const VideoBitrateAllocation& bitrate_allocation,
Erik Språng08127a92016-11-16 16:41:30 +0100297 uint32_t frame_rate) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000298 if (!inited_) {
299 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
300 }
301 if (encoder_->err) {
302 return WEBRTC_VIDEO_CODEC_ERROR;
303 }
Erik Språng08127a92016-11-16 16:41:30 +0100304 if (frame_rate < 1) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000305 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
306 }
307 // Update bit rate
Erik Språng08127a92016-11-16 16:41:30 +0100308 if (codec_.maxBitrate > 0 &&
309 bitrate_allocation.get_sum_kbps() > codec_.maxBitrate) {
310 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000311 }
Erik Språng08127a92016-11-16 16:41:30 +0100312
Erik Språng08127a92016-11-16 16:41:30 +0100313 codec_.maxFramerate = frame_rate;
asaperssona9455ab2015-07-31 06:10:09 -0700314
Sergey Silkin86684962018-03-28 19:32:37 +0200315 if (!SetSvcRates(bitrate_allocation)) {
asaperssona9455ab2015-07-31 06:10:09 -0700316 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
317 }
318
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000319 // Update encoder context
320 if (vpx_codec_enc_config_set(encoder_, config_)) {
321 return WEBRTC_VIDEO_CODEC_ERROR;
322 }
323 return WEBRTC_VIDEO_CODEC_OK;
324}
325
326int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
327 int number_of_cores,
pkasting@chromium.org4591fbd2014-11-20 22:28:14 +0000328 size_t /*max_payload_size*/) {
sprang3958ed82017-08-17 08:12:10 -0700329 if (inst == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000330 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
331 }
332 if (inst->maxFramerate < 1) {
333 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
334 }
335 // Allow zero to represent an unspecified maxBitRate
336 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
337 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
338 }
339 if (inst->width < 1 || inst->height < 1) {
340 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
341 }
342 if (number_of_cores < 1) {
343 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
344 }
hta257dc392016-10-25 09:05:06 -0700345 if (inst->VP9().numberOfTemporalLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700346 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
347 }
ilnik2a8c2f52017-02-15 02:23:28 -0800348 // libvpx probably does not support more than 3 spatial layers.
349 if (inst->VP9().numberOfSpatialLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700350 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
351 }
philipelcfc319b2015-11-10 07:17:23 -0800352
asapersson86956de2016-01-26 01:05:20 -0800353 int ret_val = Release();
354 if (ret_val < 0) {
355 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000356 }
sprang3958ed82017-08-17 08:12:10 -0700357 if (encoder_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000358 encoder_ = new vpx_codec_ctx_t;
359 }
sprang3958ed82017-08-17 08:12:10 -0700360 if (config_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000361 config_ = new vpx_codec_enc_cfg_t;
362 }
363 timestamp_ = 0;
364 if (&codec_ != inst) {
365 codec_ = *inst;
366 }
asaperssona9455ab2015-07-31 06:10:09 -0700367
Sergey Silkinb0bd03b2018-09-28 14:56:39 +0200368 force_key_frame_ = true;
369 pics_since_key_ = 0;
370
hta257dc392016-10-25 09:05:06 -0700371 num_spatial_layers_ = inst->VP9().numberOfSpatialLayers;
Niels Möller65fb4042018-04-25 14:46:06 +0200372 RTC_DCHECK_GT(num_spatial_layers_, 0);
hta257dc392016-10-25 09:05:06 -0700373 num_temporal_layers_ = inst->VP9().numberOfTemporalLayers;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200374 if (num_temporal_layers_ == 0) {
asaperssona9455ab2015-07-31 06:10:09 -0700375 num_temporal_layers_ = 1;
Sergey Silkin042661b2018-09-10 10:39:35 +0000376 }
Sergey Silkinae9e1882018-09-05 21:07:17 +0200377
Sergey Silkin96f2c972018-09-05 21:07:17 +0200378 framerate_controller_ = std::vector<FramerateController>(
379 num_spatial_layers_, FramerateController(codec_.maxFramerate));
380
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200381 is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200382
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000383 // Allocate memory for encoded image
sprang3958ed82017-08-17 08:12:10 -0700384 if (encoded_image_._buffer != nullptr) {
philipelcce46fc2015-12-21 03:04:49 -0800385 delete[] encoded_image_._buffer;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000386 }
nisseeb44b392017-04-28 07:18:05 -0700387 encoded_image_._size =
388 CalcBufferSize(VideoType::kI420, codec_.width, codec_.height);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000389 encoded_image_._buffer = new uint8_t[encoded_image_._size];
390 encoded_image_._completeFrame = true;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000391 // Populate encoder configuration with default values.
392 if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) {
393 return WEBRTC_VIDEO_CODEC_ERROR;
394 }
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700395
396 vpx_img_fmt img_fmt = VPX_IMG_FMT_NONE;
397 unsigned int bits_for_storage = 8;
398 switch (profile_) {
399 case VP9Profile::kProfile0:
400 img_fmt = VPX_IMG_FMT_I420;
401 bits_for_storage = 8;
402 config_->g_bit_depth = VPX_BITS_8;
403 config_->g_profile = 0;
404 config_->g_input_bit_depth = 8;
405 break;
406 case VP9Profile::kProfile2:
407 img_fmt = VPX_IMG_FMT_I42016;
408 bits_for_storage = 16;
409 config_->g_bit_depth = VPX_BITS_10;
410 config_->g_profile = 2;
411 config_->g_input_bit_depth = 10;
412 break;
413 }
414
415 // Creating a wrapper to the image - setting image data to nullptr. Actual
416 // pointer will be set in encode. Setting align to 1, as it is meaningless
417 // (actual memory is not allocated).
418 raw_ =
419 vpx_img_wrap(nullptr, img_fmt, codec_.width, codec_.height, 1, nullptr);
420 raw_->bit_depth = bits_for_storage;
421
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000422 config_->g_w = codec_.width;
423 config_->g_h = codec_.height;
424 config_->rc_target_bitrate = inst->startBitrate; // in kbit/s
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200425 config_->g_error_resilient = is_svc_ ? VPX_ERROR_RESILIENT_DEFAULT : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000426 // Setting the time base of the codec.
427 config_->g_timebase.num = 1;
428 config_->g_timebase.den = 90000;
429 config_->g_lag_in_frames = 0; // 0- no frame lagging
430 config_->g_threads = 1;
431 // Rate control settings.
hta257dc392016-10-25 09:05:06 -0700432 config_->rc_dropframe_thresh = inst->VP9().frameDroppingOn ? 30 : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000433 config_->rc_end_usage = VPX_CBR;
434 config_->g_pass = VPX_RC_ONE_PASS;
435 config_->rc_min_quantizer = 2;
marpan@webrtc.orgdc8a9da2015-01-27 23:08:24 +0000436 config_->rc_max_quantizer = 52;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000437 config_->rc_undershoot_pct = 50;
438 config_->rc_overshoot_pct = 50;
439 config_->rc_buf_initial_sz = 500;
440 config_->rc_buf_optimal_sz = 600;
441 config_->rc_buf_sz = 1000;
442 // Set the maximum target size of any key-frame.
443 rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz);
hta257dc392016-10-25 09:05:06 -0700444 if (inst->VP9().keyFrameInterval > 0) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000445 config_->kf_mode = VPX_KF_AUTO;
hta257dc392016-10-25 09:05:06 -0700446 config_->kf_max_dist = inst->VP9().keyFrameInterval;
Åsa Perssonff24c042015-12-04 10:58:08 +0100447 // Needs to be set (in svc mode) to get correct periodic key frame interval
448 // (will have no effect in non-svc).
449 config_->kf_min_dist = config_->kf_max_dist;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000450 } else {
451 config_->kf_mode = VPX_KF_DISABLED;
452 }
hta257dc392016-10-25 09:05:06 -0700453 config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000454 // Determine number of threads based on the image size and #cores.
philipelcce46fc2015-12-21 03:04:49 -0800455 config_->g_threads =
456 NumberOfThreads(config_->g_w, config_->g_h, number_of_cores);
asaperssona9455ab2015-07-31 06:10:09 -0700457
Marco6e89b252015-07-07 14:40:38 -0700458 cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h);
asaperssona9455ab2015-07-31 06:10:09 -0700459
hta257dc392016-10-25 09:05:06 -0700460 is_flexible_mode_ = inst->VP9().flexibleMode;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200461
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200462 if (num_temporal_layers_ == 1) {
asaperssona9455ab2015-07-31 06:10:09 -0700463 gof_.SetGofInfoVP9(kTemporalStructureMode1);
464 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING;
465 config_->ts_number_layers = 1;
466 config_->ts_rate_decimator[0] = 1;
467 config_->ts_periodicity = 1;
468 config_->ts_layer_id[0] = 0;
469 } else if (num_temporal_layers_ == 2) {
470 gof_.SetGofInfoVP9(kTemporalStructureMode2);
471 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
472 config_->ts_number_layers = 2;
473 config_->ts_rate_decimator[0] = 2;
474 config_->ts_rate_decimator[1] = 1;
475 config_->ts_periodicity = 2;
476 config_->ts_layer_id[0] = 0;
477 config_->ts_layer_id[1] = 1;
478 } else if (num_temporal_layers_ == 3) {
479 gof_.SetGofInfoVP9(kTemporalStructureMode3);
480 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212;
481 config_->ts_number_layers = 3;
482 config_->ts_rate_decimator[0] = 4;
483 config_->ts_rate_decimator[1] = 2;
484 config_->ts_rate_decimator[2] = 1;
485 config_->ts_periodicity = 4;
486 config_->ts_layer_id[0] = 0;
487 config_->ts_layer_id[1] = 2;
488 config_->ts_layer_id[2] = 1;
489 config_->ts_layer_id[3] = 2;
490 } else {
491 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
492 }
493
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200494 inter_layer_pred_ = inst->VP9().interLayerPred;
495
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200496 ref_buf_.clear();
497
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000498 return InitAndSetControlSettings(inst);
499}
500
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000501int VP9EncoderImpl::NumberOfThreads(int width,
502 int height,
503 int number_of_cores) {
504 // Keep the number of encoder threads equal to the possible number of column
505 // tiles, which is (1, 2, 4, 8). See comments below for VP9E_SET_TILE_COLUMNS.
506 if (width * height >= 1280 * 720 && number_of_cores > 4) {
507 return 4;
jianj23173a32017-07-12 16:11:09 -0700508 } else if (width * height >= 640 * 360 && number_of_cores > 2) {
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000509 return 2;
510 } else {
Yves Gerey665174f2018-06-19 15:03:05 +0200511// Use 2 threads for low res on ARM.
Jerome Jiang831af372017-12-05 10:44:35 -0800512#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
513 defined(WEBRTC_ANDROID)
514 if (width * height >= 320 * 180 && number_of_cores > 2) {
515 return 2;
516 }
517#endif
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000518 // 1 thread less than VGA.
519 return 1;
520 }
521}
522
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000523int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) {
Åsa Perssonff24c042015-12-04 10:58:08 +0100524 // Set QP-min/max per spatial and temporal layer.
525 int tot_num_layers = num_spatial_layers_ * num_temporal_layers_;
526 for (int i = 0; i < tot_num_layers; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800527 svc_params_.max_quantizers[i] = config_->rc_max_quantizer;
528 svc_params_.min_quantizers[i] = config_->rc_min_quantizer;
Åsa Perssonff24c042015-12-04 10:58:08 +0100529 }
asaperssona9455ab2015-07-31 06:10:09 -0700530 config_->ss_number_layers = num_spatial_layers_;
sprangce4aef12015-11-02 07:23:20 -0800531 if (ExplicitlyConfiguredSpatialLayers()) {
532 for (int i = 0; i < num_spatial_layers_; ++i) {
533 const auto& layer = codec_.spatialLayers[i];
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200534 RTC_CHECK_GT(layer.width, 0);
Sergey Silkin13e74342018-03-02 12:28:00 +0100535 const int scale_factor = codec_.width / layer.width;
536 RTC_DCHECK_GT(scale_factor, 0);
537
538 // Ensure scaler factor is integer.
539 if (scale_factor * layer.width != codec_.width) {
540 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
541 }
542
543 // Ensure scale factor is the same in both dimensions.
544 if (scale_factor * layer.height != codec_.height) {
545 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
546 }
547
548 // Ensure scale factor is power of two.
549 const bool is_pow_of_two = (scale_factor & (scale_factor - 1)) == 0;
550 if (!is_pow_of_two) {
551 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
552 }
553
554 svc_params_.scaling_factor_num[i] = 1;
555 svc_params_.scaling_factor_den[i] = scale_factor;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200556
557 RTC_DCHECK_GT(codec_.spatialLayers[i].maxFramerate, 0);
558 RTC_DCHECK_LE(codec_.spatialLayers[i].maxFramerate, codec_.maxFramerate);
559 if (i > 0) {
560 // Frame rate of high spatial layer is supposed to be equal or higher
561 // than frame rate of low spatial layer.
562 RTC_DCHECK_GE(codec_.spatialLayers[i].maxFramerate,
563 codec_.spatialLayers[i - 1].maxFramerate);
564 }
sprangce4aef12015-11-02 07:23:20 -0800565 }
566 } else {
567 int scaling_factor_num = 256;
568 for (int i = num_spatial_layers_ - 1; i >= 0; --i) {
sprangce4aef12015-11-02 07:23:20 -0800569 // 1:2 scaling in each dimension.
johannkoenig8225c402017-01-26 13:23:44 -0800570 svc_params_.scaling_factor_num[i] = scaling_factor_num;
571 svc_params_.scaling_factor_den[i] = 256;
sprangce4aef12015-11-02 07:23:20 -0800572 }
asaperssona9455ab2015-07-31 06:10:09 -0700573 }
574
Sergey Silkin86684962018-03-28 19:32:37 +0200575 SvcRateAllocator init_allocator(codec_);
Erik Språng566124a2018-04-23 12:32:22 +0200576 VideoBitrateAllocation allocation = init_allocator.GetAllocation(
Sergey Silkin86684962018-03-28 19:32:37 +0200577 inst->startBitrate * 1000, inst->maxFramerate);
578 if (!SetSvcRates(allocation)) {
asaperssona9455ab2015-07-31 06:10:09 -0700579 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
580 }
581
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700582 const vpx_codec_err_t rv = vpx_codec_enc_init(
583 encoder_, vpx_codec_vp9_cx(), config_,
584 config_->g_bit_depth == VPX_BITS_8 ? 0 : VPX_CODEC_USE_HIGHBITDEPTH);
585 if (rv != VPX_CODEC_OK) {
586 RTC_LOG(LS_ERROR) << "Init error: " << vpx_codec_err_to_string(rv);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000587 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
588 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000589 vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_);
590 vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT,
591 rc_max_intra_target_);
592 vpx_codec_control(encoder_, VP9E_SET_AQ_MODE,
hta257dc392016-10-25 09:05:06 -0700593 inst->VP9().adaptiveQpMode ? 3 : 0);
asaperssona9455ab2015-07-31 06:10:09 -0700594
jianj822e5932017-07-12 16:09:58 -0700595 vpx_codec_control(encoder_, VP9E_SET_FRAME_PARALLEL_DECODING, 0);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200596
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200597 if (is_svc_) {
598 vpx_codec_control(encoder_, VP9E_SET_SVC, 1);
599 vpx_codec_control(encoder_, VP9E_SET_SVC_PARAMETERS, &svc_params_);
asaperssona9455ab2015-07-31 06:10:09 -0700600 }
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200601
602 if (num_spatial_layers_ > 1) {
603 switch (inter_layer_pred_) {
604 case InterLayerPredMode::kOn:
605 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 0);
606 break;
607 case InterLayerPredMode::kOff:
608 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 1);
609 break;
610 case InterLayerPredMode::kOnKeyPic:
611 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 2);
612 break;
613 default:
614 RTC_NOTREACHED();
615 }
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200616
Sergey Silkinee203362018-05-30 11:34:08 +0200617 // Configure encoder to drop entire superframe whenever it needs to drop
618 // a layer. This mode is prefered over per-layer dropping which causes
619 // quality flickering and is not compatible with RTP non-flexible mode.
620 vpx_svc_frame_drop_t svc_drop_frame;
621 memset(&svc_drop_frame, 0, sizeof(svc_drop_frame));
622 svc_drop_frame.framedrop_mode = FULL_SUPERFRAME_DROP;
Sergey Silkind45b3452018-05-31 16:00:24 +0200623 svc_drop_frame.max_consec_drop = std::numeric_limits<int>::max();
Sergey Silkinee203362018-05-30 11:34:08 +0200624 for (size_t i = 0; i < num_spatial_layers_; ++i) {
625 svc_drop_frame.framedrop_thresh[i] = config_->rc_dropframe_thresh;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200626 }
Sergey Silkinee203362018-05-30 11:34:08 +0200627 vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200628 }
629
asaperssona9455ab2015-07-31 06:10:09 -0700630 // Register callback for getting each spatial layer.
631 vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = {
philipelcce46fc2015-12-21 03:04:49 -0800632 VP9EncoderImpl::EncoderOutputCodedPacketCallback,
633 reinterpret_cast<void*>(this)};
634 vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK,
635 reinterpret_cast<void*>(&cbp));
asaperssona9455ab2015-07-31 06:10:09 -0700636
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000637 // Control function to set the number of column tiles in encoding a frame, in
638 // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns.
639 // The number tile columns will be capped by the encoder based on image size
640 // (minimum width of tile column is 256 pixels, maximum is 4096).
641 vpx_codec_control(encoder_, VP9E_SET_TILE_COLUMNS, (config_->g_threads >> 1));
jianjcb5d1152017-03-28 23:56:08 -0700642
643 // Turn on row-based multithreading.
644 vpx_codec_control(encoder_, VP9E_SET_ROW_MT, 1);
jianj6bf57e32017-06-05 13:43:49 -0700645
Alex Glaznevfecb7c32016-03-31 14:23:27 -0700646#if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
Yves Gerey665174f2018-06-19 15:03:05 +0200647 !defined(ANDROID)
jianj6bf57e32017-06-05 13:43:49 -0700648 // Do not enable the denoiser on ARM since optimization is pending.
649 // Denoiser is on by default on other platforms.
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000650 vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY,
hta257dc392016-10-25 09:05:06 -0700651 inst->VP9().denoisingOn ? 1 : 0);
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000652#endif
jianj6bf57e32017-06-05 13:43:49 -0700653
Niels Möllere3cf3d02018-06-13 11:52:16 +0200654 if (codec_.mode == VideoCodecMode::kScreensharing) {
ivica242d6382015-09-04 06:13:23 -0700655 // Adjust internal parameters to screen content.
656 vpx_codec_control(encoder_, VP9E_SET_TUNE_CONTENT, 1);
ivica242d6382015-09-04 06:13:23 -0700657 }
Marco2520e722015-09-16 14:05:00 -0700658 // Enable encoder skip of static/low content blocks.
659 vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000660 inited_ = true;
661 return WEBRTC_VIDEO_CODEC_OK;
662}
663
664uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) {
665 // Set max to the optimal buffer level (normalized by target BR),
666 // and scaled by a scale_par.
667 // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps].
668 // This value is presented in percentage of perFrameBw:
669 // perFrameBw = targetBR[Kbps] * 1000 / framerate.
670 // The target in % is as follows:
671 float scale_par = 0.5;
672 uint32_t target_pct =
673 optimal_buffer_size * scale_par * codec_.maxFramerate / 10;
674 // Don't go below 3 times the per frame bandwidth.
675 const uint32_t min_intra_size = 300;
philipelcce46fc2015-12-21 03:04:49 -0800676 return (target_pct < min_intra_size) ? min_intra_size : target_pct;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000677}
678
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700679int VP9EncoderImpl::Encode(const VideoFrame& input_image,
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000680 const CodecSpecificInfo* codec_specific_info,
pbos22993e12015-10-19 02:39:06 -0700681 const std::vector<FrameType>* frame_types) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000682 if (!inited_) {
683 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
684 }
sprang3958ed82017-08-17 08:12:10 -0700685 if (encoded_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000686 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
687 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200688
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000689 // We only support one stream at the moment.
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200690 if (frame_types && !frame_types->empty()) {
691 if ((*frame_types)[0] == kVideoFrameKey) {
692 force_key_frame_ = true;
693 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000694 }
Sergey Silkind902d582018-05-18 17:31:19 +0200695
Niels Möllere3cf3d02018-06-13 11:52:16 +0200696 if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) {
Sergey Silkin96f2c972018-09-05 21:07:17 +0200697 // Skip encoding spatial layer frames if their target frame rate is lower
698 // than actual input frame rate.
699 vpx_svc_layer_id_t layer_id = {0};
700 const size_t gof_idx = (pics_since_key_ + 1) % gof_.num_frames_in_gof;
701 layer_id.temporal_layer_id = gof_.temporal_idx[gof_idx];
702
703 const uint32_t frame_timestamp_ms =
704 1000 * input_image.timestamp() / kVideoPayloadTypeFrequency;
705
706 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
707 if (framerate_controller_[sl_idx].DropFrame(frame_timestamp_ms)) {
708 ++layer_id.spatial_layer_id;
709 } else {
710 break;
711 }
712 }
713
714 RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
715 if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
716 // Drop entire picture.
Sergey Silkind902d582018-05-18 17:31:19 +0200717 return WEBRTC_VIDEO_CODEC_OK;
718 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200719
720 vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
Sergey Silkind902d582018-05-18 17:31:19 +0200721 }
722
kwiberg352444f2016-11-28 15:58:53 -0800723 RTC_DCHECK_EQ(input_image.width(), raw_->d_w);
724 RTC_DCHECK_EQ(input_image.height(), raw_->d_h);
asaperssona9455ab2015-07-31 06:10:09 -0700725
726 // Set input image for use in the callback.
727 // This was necessary since you need some information from input_image.
728 // You can save only the necessary information (such as timestamp) instead of
729 // doing this.
730 input_image_ = &input_image;
731
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700732 // Keep reference to buffer until encode completes.
733 rtc::scoped_refptr<I420BufferInterface> i420_buffer;
734 rtc::scoped_refptr<I010BufferInterface> i010_buffer;
735 switch (profile_) {
736 case VP9Profile::kProfile0: {
737 i420_buffer = input_image.video_frame_buffer()->ToI420();
738 // Image in vpx_image_t format.
739 // Input image is const. VPX's raw image is not defined as const.
740 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
741 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU());
742 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV());
743 raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
744 raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU();
745 raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
746 break;
747 }
748 case VP9Profile::kProfile2: {
749 // We can inject kI010 frames directly for encode. All other formats
750 // should be converted to it.
751 switch (input_image.video_frame_buffer()->type()) {
752 case VideoFrameBuffer::Type::kI010: {
753 i010_buffer = input_image.video_frame_buffer()->GetI010();
754 break;
755 }
756 default: {
757 i010_buffer =
758 I010Buffer::Copy(*input_image.video_frame_buffer()->ToI420());
759 }
760 }
761 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(
762 reinterpret_cast<const uint8_t*>(i010_buffer->DataY()));
763 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(
764 reinterpret_cast<const uint8_t*>(i010_buffer->DataU()));
765 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(
766 reinterpret_cast<const uint8_t*>(i010_buffer->DataV()));
767 raw_->stride[VPX_PLANE_Y] = i010_buffer->StrideY() * 2;
768 raw_->stride[VPX_PLANE_U] = i010_buffer->StrideU() * 2;
769 raw_->stride[VPX_PLANE_V] = i010_buffer->StrideV() * 2;
770 break;
771 }
772 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000773
philipelcfc319b2015-11-10 07:17:23 -0800774 vpx_enc_frame_flags_t flags = 0;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200775 if (force_key_frame_) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000776 flags = VPX_EFLAG_FORCE_KF;
777 }
philipelcfc319b2015-11-10 07:17:23 -0800778
Sergey Silkin390f3582018-10-01 17:13:50 +0200779 if (external_ref_control_) {
780 vpx_svc_ref_frame_config_t ref_config = SetReferences(force_key_frame_);
Sergey Silkina85995a2018-10-02 10:28:10 +0200781
782 if (VideoCodecMode::kScreensharing == codec_.mode) {
783 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
784 ref_config.duration[sl_idx] = static_cast<int64_t>(
785 90000 / framerate_controller_[sl_idx].GetTargetRate());
786 }
787 }
788
Sergey Silkin390f3582018-10-01 17:13:50 +0200789 vpx_codec_control(encoder_, VP9E_SET_SVC_REF_FRAME_CONFIG, &ref_config);
790 }
791
Sergey Silkin96f2c972018-09-05 21:07:17 +0200792 // TODO(ssilkin): Frame duration should be specified per spatial layer
793 // since their frame rate can be different. For now calculate frame duration
794 // based on target frame rate of the highest spatial layer, which frame rate
795 // is supposed to be equal or higher than frame rate of low spatial layers.
796 // Also, timestamp should represent actual time passed since previous frame
797 // (not 'expected' time). Then rate controller can drain buffer more
798 // accurately.
799 RTC_DCHECK_GE(framerate_controller_.size(), num_active_spatial_layers_);
Sergey Silkinf87bb462018-09-17 09:37:37 +0200800 float target_framerate_fps =
801 (codec_.mode == VideoCodecMode::kScreensharing)
802 ? framerate_controller_[num_active_spatial_layers_ - 1]
803 .GetTargetRate()
804 : codec_.maxFramerate;
805 uint32_t duration = static_cast<uint32_t>(90000 / target_framerate_fps);
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700806 const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_,
807 duration, flags, VPX_DL_REALTIME);
808 if (rv != VPX_CODEC_OK) {
809 RTC_LOG(LS_ERROR) << "Encoding error: " << vpx_codec_err_to_string(rv)
810 << "\n"
811 << "Details: " << vpx_codec_error(encoder_) << "\n"
812 << vpx_codec_error_detail(encoder_);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000813 return WEBRTC_VIDEO_CODEC_ERROR;
814 }
815 timestamp_ += duration;
asaperssona9455ab2015-07-31 06:10:09 -0700816
Sergey Silkinbc0f0d32018-04-24 21:29:14 +0200817 const bool end_of_picture = true;
818 DeliverBufferedFrame(end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200819
asaperssona9455ab2015-07-31 06:10:09 -0700820 return WEBRTC_VIDEO_CODEC_OK;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000821}
822
823void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
Niels Möllerd3b8c632018-08-27 15:33:42 +0200824 absl::optional<int>* spatial_idx,
philipelcce46fc2015-12-21 03:04:49 -0800825 const vpx_codec_cx_pkt& pkt,
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200826 uint32_t timestamp,
827 bool first_frame_in_picture) {
sprang3958ed82017-08-17 08:12:10 -0700828 RTC_CHECK(codec_specific != nullptr);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000829 codec_specific->codecType = kVideoCodecVP9;
philipelcce46fc2015-12-21 03:04:49 -0800830 CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9);
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200831
832 vp9_info->first_frame_in_picture = first_frame_in_picture;
Sergey Silkin738b7e92018-08-23 16:37:27 +0200833 vp9_info->flexible_mode = is_flexible_mode_;
hta257dc392016-10-25 09:05:06 -0700834 vp9_info->ss_data_available =
Sergey Silkin738b7e92018-08-23 16:37:27 +0200835 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
asaperssona9455ab2015-07-31 06:10:09 -0700836
837 vpx_svc_layer_id_t layer_id = {0};
838 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
839
sprang3958ed82017-08-17 08:12:10 -0700840 RTC_CHECK_GT(num_temporal_layers_, 0);
“Michael23c5a992018-06-21 11:07:21 -0500841 RTC_CHECK_GT(num_active_spatial_layers_, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700842 if (num_temporal_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700843 RTC_CHECK_EQ(layer_id.temporal_layer_id, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700844 vp9_info->temporal_idx = kNoTemporalIdx;
845 } else {
846 vp9_info->temporal_idx = layer_id.temporal_layer_id;
847 }
“Michael23c5a992018-06-21 11:07:21 -0500848 if (num_active_spatial_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700849 RTC_CHECK_EQ(layer_id.spatial_layer_id, 0);
Niels Möllerd3b8c632018-08-27 15:33:42 +0200850 *spatial_idx = absl::nullopt;
asaperssona9455ab2015-07-31 06:10:09 -0700851 } else {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200852 *spatial_idx = layer_id.spatial_layer_id;
asaperssona9455ab2015-07-31 06:10:09 -0700853 }
854 if (layer_id.spatial_layer_id != 0) {
855 vp9_info->ss_data_available = false;
856 }
857
asaperssona9455ab2015-07-31 06:10:09 -0700858 // TODO(asapersson): this info has to be obtained from the encoder.
asaperssoncb50c962015-11-18 01:58:55 -0800859 vp9_info->temporal_up_switch = false;
asaperssona9455ab2015-07-31 06:10:09 -0700860
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200861 if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
862 pics_since_key_ = 0;
863 } else if (first_frame_in_picture) {
864 ++pics_since_key_;
asaperssona9455ab2015-07-31 06:10:09 -0700865 }
866
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200867 const bool is_key_pic = (pics_since_key_ == 0);
868 const bool is_inter_layer_pred_allowed =
869 (inter_layer_pred_ == InterLayerPredMode::kOn ||
870 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic));
871
872 // Always set inter_layer_predicted to true on high layer frame if inter-layer
873 // prediction (ILP) is allowed even if encoder didn't actually use it.
874 // Setting inter_layer_predicted to false would allow receiver to decode high
875 // layer frame without decoding low layer frame. If that would happen (e.g.
876 // if low layer frame is lost) then receiver won't be able to decode next high
877 // layer frame which uses ILP.
878 vp9_info->inter_layer_predicted =
879 first_frame_in_picture ? false : is_inter_layer_pred_allowed;
880
Sergey Silkinc5f152d2018-09-26 13:28:50 +0200881 // Mark all low spatial layer frames as references (not just frames of
882 // active low spatial layers) if inter-layer prediction is enabled since
883 // these frames are indirect references of high spatial layer, which can
884 // later be enabled without key frame.
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200885 vp9_info->non_ref_for_inter_layer_pred =
Sergey Silkinc5f152d2018-09-26 13:28:50 +0200886 !is_inter_layer_pred_allowed ||
887 layer_id.spatial_layer_id + 1 == num_spatial_layers_;
asapersson00ac85e2015-11-11 05:30:48 -0800888
ivica7f6a6fc2015-09-08 02:40:29 -0700889 // Always populate this, so that the packetizer can properly set the marker
890 // bit.
“Michael23c5a992018-06-21 11:07:21 -0500891 vp9_info->num_spatial_layers = num_active_spatial_layers_;
philipelcfc319b2015-11-10 07:17:23 -0800892
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200893 vp9_info->num_ref_pics = 0;
894 if (vp9_info->flexible_mode) {
895 vp9_info->gof_idx = kNoGofIdx;
896 FillReferenceIndices(pkt, pics_since_key_, vp9_info->inter_layer_predicted,
897 vp9_info);
Sergey Silkin390f3582018-10-01 17:13:50 +0200898 // TODO(webrtc:9794): Add fake reference to empty reference list to
899 // workaround the frame buffer issue on receiver.
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200900 } else {
901 vp9_info->gof_idx =
902 static_cast<uint8_t>(pics_since_key_ % gof_.num_frames_in_gof);
903 vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx];
904 vp9_info->num_ref_pics = gof_.num_ref_pics[vp9_info->gof_idx];
905 }
906
907 vp9_info->inter_pic_predicted = (!is_key_pic && vp9_info->num_ref_pics > 0);
908
asaperssona9455ab2015-07-31 06:10:09 -0700909 if (vp9_info->ss_data_available) {
asaperssona9455ab2015-07-31 06:10:09 -0700910 vp9_info->spatial_layer_resolution_present = true;
“Michael23c5a992018-06-21 11:07:21 -0500911 for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
Yves Gerey665174f2018-06-19 15:03:05 +0200912 vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -0800913 svc_params_.scaling_factor_den[i];
Yves Gerey665174f2018-06-19 15:03:05 +0200914 vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -0800915 svc_params_.scaling_factor_den[i];
asaperssona9455ab2015-07-31 06:10:09 -0700916 }
Sergey Silkin738b7e92018-08-23 16:37:27 +0200917 if (vp9_info->flexible_mode) {
918 vp9_info->gof.num_frames_in_gof = 0;
919 } else {
asaperssona9455ab2015-07-31 06:10:09 -0700920 vp9_info->gof.CopyGofInfoVP9(gof_);
921 }
922 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000923}
924
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200925void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
926 const size_t pic_num,
927 const bool inter_layer_predicted,
928 CodecSpecificInfoVP9* vp9_info) {
929 vpx_svc_layer_id_t layer_id = {0};
930 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
931
Sergey Silkin76be2952018-08-21 12:30:33 +0200932 const bool is_key_frame =
933 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200934
935 std::vector<RefFrameBuffer> ref_buf_list;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200936
Sergey Silkin76be2952018-08-21 12:30:33 +0200937 if (is_svc_) {
938 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
939 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200940
Sergey Silkin76be2952018-08-21 12:30:33 +0200941 if (enc_layer_conf.reference_last[layer_id.spatial_layer_id]) {
942 const size_t fb_idx =
943 enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id];
944 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +0200945 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
946 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
947 ref_buf_list.push_back(ref_buf_.at(fb_idx));
948 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200949 }
950
951 if (enc_layer_conf.reference_alt_ref[layer_id.spatial_layer_id]) {
952 const size_t fb_idx =
953 enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id];
954 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +0200955 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
956 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
957 ref_buf_list.push_back(ref_buf_.at(fb_idx));
958 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200959 }
960
961 if (enc_layer_conf.reference_golden[layer_id.spatial_layer_id]) {
962 const size_t fb_idx =
963 enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id];
964 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +0200965 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
966 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
967 ref_buf_list.push_back(ref_buf_.at(fb_idx));
968 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200969 }
970 } else if (!is_key_frame) {
971 RTC_DCHECK_EQ(num_spatial_layers_, 1);
972 RTC_DCHECK_EQ(num_temporal_layers_, 1);
973 // In non-SVC mode encoder doesn't provide reference list. Assume each frame
974 // refers previous one, which is stored in buffer 0.
975 ref_buf_list.push_back(ref_buf_.at(0));
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200976 }
977
978 size_t max_ref_temporal_layer_id = 0;
979
980 vp9_info->num_ref_pics = 0;
981 for (const RefFrameBuffer& ref_buf : ref_buf_list) {
982 RTC_DCHECK_LE(ref_buf.pic_num, pic_num);
983 if (ref_buf.pic_num < pic_num) {
984 if (inter_layer_pred_ != InterLayerPredMode::kOn) {
985 // RTP spec limits temporal prediction to the same spatial layer.
986 // It is safe to ignore this requirement if inter-layer prediction is
987 // enabled for all frames when all base frames are relayed to receiver.
988 RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id);
989 }
990 RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id);
991
992 const size_t p_diff = pic_num - ref_buf.pic_num;
993 RTC_DCHECK_LE(p_diff, 127UL);
994
995 vp9_info->p_diff[vp9_info->num_ref_pics] = static_cast<uint8_t>(p_diff);
996 ++vp9_info->num_ref_pics;
997
998 max_ref_temporal_layer_id =
999 std::max(max_ref_temporal_layer_id, ref_buf.temporal_layer_id);
1000 } else {
1001 RTC_DCHECK(inter_layer_predicted);
1002 // RTP spec only allows to use previous spatial layer for inter-layer
1003 // prediction.
1004 RTC_DCHECK_EQ(ref_buf.spatial_layer_id + 1, layer_id.spatial_layer_id);
1005 }
1006 }
1007
1008 vp9_info->temporal_up_switch =
1009 (max_ref_temporal_layer_id <
1010 static_cast<size_t>(layer_id.temporal_layer_id));
1011}
1012
1013void VP9EncoderImpl::UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
1014 const size_t pic_num) {
1015 vpx_svc_layer_id_t layer_id = {0};
1016 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1017
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001018 const bool is_key_frame =
1019 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
1020
1021 RefFrameBuffer frame_buf(pic_num, layer_id.spatial_layer_id,
1022 layer_id.temporal_layer_id);
1023
1024 if (is_key_frame && layer_id.spatial_layer_id == 0) {
1025 // Key frame updates all ref buffers.
1026 for (size_t i = 0; i < kNumVp9Buffers; ++i) {
1027 ref_buf_[i] = frame_buf;
1028 }
Sergey Silkin76be2952018-08-21 12:30:33 +02001029 } else if (is_svc_) {
1030 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
1031 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
1032
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001033 if (enc_layer_conf.update_last[layer_id.spatial_layer_id]) {
1034 ref_buf_[enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id]] =
1035 frame_buf;
1036 }
1037
1038 if (enc_layer_conf.update_alt_ref[layer_id.spatial_layer_id]) {
1039 ref_buf_[enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id]] =
1040 frame_buf;
1041 }
1042
1043 if (enc_layer_conf.update_golden[layer_id.spatial_layer_id]) {
1044 ref_buf_[enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id]] =
1045 frame_buf;
1046 }
Sergey Silkin76be2952018-08-21 12:30:33 +02001047 } else {
1048 RTC_DCHECK_EQ(num_spatial_layers_, 1);
1049 RTC_DCHECK_EQ(num_temporal_layers_, 1);
1050 // In non-svc mode encoder doesn't provide reference list. Assume each frame
1051 // is reference and stored in buffer 0.
1052 ref_buf_[0] = frame_buf;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001053 }
1054}
1055
Sergey Silkin390f3582018-10-01 17:13:50 +02001056vpx_svc_ref_frame_config_t VP9EncoderImpl::SetReferences(bool is_key_pic) {
1057 // kRefBufIdx, kUpdBufIdx need to be updated to support longer GOFs.
1058 RTC_DCHECK_LE(gof_.num_frames_in_gof, 4);
1059
1060 vpx_svc_ref_frame_config_t ref_config;
1061 memset(&ref_config, 0, sizeof(ref_config));
1062
1063 const size_t num_temporal_refs = std::max(1, num_temporal_layers_ - 1);
1064 const bool is_inter_layer_pred_allowed =
1065 inter_layer_pred_ == InterLayerPredMode::kOn ||
1066 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic);
1067 absl::optional<int> last_updated_buf_idx;
1068
1069 // Put temporal reference to LAST and spatial reference to GOLDEN. Update
1070 // frame buffer (i.e. store encoded frame) if current frame is a temporal
1071 // reference (i.e. it belongs to a low temporal layer) or it is a spatial
1072 // reference. In later case, always store spatial reference in the last
1073 // reference frame buffer.
1074 // For the case of 3 temporal and 3 spatial layers we need 6 frame buffers
1075 // for temporal references plus 1 buffer for spatial reference. 7 buffers
1076 // in total.
1077
1078 for (size_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
1079 const size_t gof_idx = pics_since_key_ % gof_.num_frames_in_gof;
1080
1081 if (!is_key_pic) {
1082 // Set up temporal reference.
1083 const int buf_idx = sl_idx * num_temporal_refs + kRefBufIdx[gof_idx];
1084
1085 // Last reference frame buffer is reserved for spatial reference. It is
1086 // not supposed to be used for temporal prediction.
1087 RTC_DCHECK_LT(buf_idx, kNumVp9Buffers - 1);
1088
1089 // Sanity check that reference picture number is smaller than current
1090 // picture number.
1091 const size_t curr_pic_num = pics_since_key_ + 1;
1092 RTC_DCHECK_LT(ref_buf_[buf_idx].pic_num, curr_pic_num);
1093 const size_t pid_diff = curr_pic_num - ref_buf_[buf_idx].pic_num;
1094
1095 // Below code assumes single temporal referecence.
1096 RTC_DCHECK_EQ(gof_.num_ref_pics[gof_idx], 1);
1097 if (pid_diff == gof_.pid_diff[gof_idx][0]) {
1098 ref_config.lst_fb_idx[sl_idx] = buf_idx;
1099 ref_config.reference_last[sl_idx] = 1;
1100 } else {
1101 // This reference doesn't match with one specified by GOF. This can
1102 // only happen if spatial layer is enabled dynamically without key
1103 // frame. Spatial prediction is supposed to be enabled in this case.
1104 RTC_DCHECK(is_inter_layer_pred_allowed);
1105 }
1106 }
1107
1108 if (is_inter_layer_pred_allowed && sl_idx > 0) {
1109 // Set up spatial reference.
1110 RTC_DCHECK(last_updated_buf_idx);
1111 ref_config.gld_fb_idx[sl_idx] = *last_updated_buf_idx;
1112 ref_config.reference_golden[sl_idx] = 1;
1113 } else {
1114 RTC_DCHECK(ref_config.reference_last[sl_idx] != 0 || sl_idx == 0 ||
1115 inter_layer_pred_ == InterLayerPredMode::kOff);
1116 }
1117
1118 last_updated_buf_idx.reset();
1119
1120 if (gof_.temporal_idx[gof_idx] <= num_temporal_layers_ - 1) {
1121 last_updated_buf_idx = sl_idx * num_temporal_refs + kUpdBufIdx[gof_idx];
1122
1123 // Ensure last frame buffer is not used for temporal prediction (it is
1124 // reserved for spatial reference).
1125 RTC_DCHECK_LT(*last_updated_buf_idx, kNumVp9Buffers - 1);
1126 } else if (is_inter_layer_pred_allowed) {
1127 last_updated_buf_idx = kNumVp9Buffers - 1;
1128 }
1129
1130 if (last_updated_buf_idx) {
1131 ref_config.update_buffer_slot[sl_idx] = 1 << *last_updated_buf_idx;
1132 }
1133 }
1134
1135 return ref_config;
1136}
1137
asaperssona9455ab2015-07-31 06:10:09 -07001138int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) {
asapersson86956de2016-01-26 01:05:20 -08001139 RTC_DCHECK_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT);
asaperssona9455ab2015-07-31 06:10:09 -07001140
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001141 if (pkt->data.frame.sz == 0) {
1142 // Ignore dropped frame.
1143 return WEBRTC_VIDEO_CODEC_OK;
1144 }
1145
Sergey Silkin07f80cc2018-04-09 13:11:59 +02001146 vpx_svc_layer_id_t layer_id = {0};
1147 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1148
1149 const bool first_frame_in_picture = encoded_image_._length == 0;
1150 // Ensure we don't buffer layers of previous picture (superframe).
1151 RTC_DCHECK(first_frame_in_picture || layer_id.spatial_layer_id > 0);
1152
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001153 const bool end_of_picture = false;
1154 DeliverBufferedFrame(end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001155
asaperssond9f641e2016-01-21 01:11:35 -08001156 if (pkt->data.frame.sz > encoded_image_._size) {
1157 delete[] encoded_image_._buffer;
1158 encoded_image_._size = pkt->data.frame.sz;
1159 encoded_image_._buffer = new uint8_t[encoded_image_._size];
1160 }
asapersson86956de2016-01-26 01:05:20 -08001161 memcpy(encoded_image_._buffer, pkt->data.frame.buf, pkt->data.frame.sz);
1162 encoded_image_._length = pkt->data.frame.sz;
asaperssond9f641e2016-01-21 01:11:35 -08001163
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001164 const bool is_key_frame =
1165 (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
1166 // Ensure encoder issued key frame on request.
1167 RTC_DCHECK(is_key_frame || !force_key_frame_);
1168
asaperssona9455ab2015-07-31 06:10:09 -07001169 // Check if encoded frame is a key frame.
asapersson86956de2016-01-26 01:05:20 -08001170 encoded_image_._frameType = kVideoFrameDelta;
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001171 if (is_key_frame) {
Peter Boström49e196a2015-10-23 15:58:18 +02001172 encoded_image_._frameType = kVideoFrameKey;
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001173 force_key_frame_ = false;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001174 }
asapersson86956de2016-01-26 01:05:20 -08001175 RTC_DCHECK_LE(encoded_image_._length, encoded_image_._size);
1176
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001177 memset(&codec_specific_, 0, sizeof(codec_specific_));
Niels Möllerd3b8c632018-08-27 15:33:42 +02001178 absl::optional<int> spatial_index;
1179 PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt,
1180 input_image_->timestamp(), first_frame_in_picture);
1181 encoded_image_.SetSpatialIndex(spatial_index);
asaperssona9455ab2015-07-31 06:10:09 -07001182
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001183 if (is_flexible_mode_) {
1184 UpdateReferenceBuffers(*pkt, pics_since_key_);
1185 }
1186
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001187 TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length);
Niels Möller23775882018-08-16 10:24:12 +02001188 encoded_image_.SetTimestamp(input_image_->timestamp());
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001189 encoded_image_.capture_time_ms_ = input_image_->render_time_ms();
1190 encoded_image_.rotation_ = input_image_->rotation();
Niels Möllere3cf3d02018-06-13 11:52:16 +02001191 encoded_image_.content_type_ = (codec_.mode == VideoCodecMode::kScreensharing)
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001192 ? VideoContentType::SCREENSHARE
1193 : VideoContentType::UNSPECIFIED;
1194 encoded_image_._encodedHeight =
1195 pkt->data.frame.height[layer_id.spatial_layer_id];
1196 encoded_image_._encodedWidth =
1197 pkt->data.frame.width[layer_id.spatial_layer_id];
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +02001198 encoded_image_.timing_.flags = VideoSendTiming::kInvalid;
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001199 int qp = -1;
1200 vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp);
1201 encoded_image_.qp_ = qp;
Johannes Kron4749e4e2018-11-21 10:18:18 +01001202 encoded_image_.SetColorSpace(input_image_->color_space());
ilnik04f4d122017-06-19 07:18:55 -07001203
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001204 return WEBRTC_VIDEO_CODEC_OK;
1205}
1206
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001207void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) {
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001208 if (encoded_image_._length > 0) {
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001209 codec_specific_.codecSpecific.VP9.end_of_picture = end_of_picture;
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001210
1211 // No data partitioning in VP9, so 1 partition only.
1212 int part_idx = 0;
1213 RTPFragmentationHeader frag_info;
1214 frag_info.VerifyAndAllocateFragmentationHeader(1);
1215 frag_info.fragmentationOffset[part_idx] = 0;
1216 frag_info.fragmentationLength[part_idx] = encoded_image_._length;
1217 frag_info.fragmentationPlType[part_idx] = 0;
1218 frag_info.fragmentationTimeDiff[part_idx] = 0;
1219
1220 encoded_complete_callback_->OnEncodedImage(encoded_image_, &codec_specific_,
1221 &frag_info);
1222 encoded_image_._length = 0;
Sergey Silkind902d582018-05-18 17:31:19 +02001223
Sergey Silkin96f2c972018-09-05 21:07:17 +02001224 if (codec_.mode == VideoCodecMode::kScreensharing) {
1225 const uint8_t spatial_idx = encoded_image_.SpatialIndex().value_or(0);
1226 const uint32_t frame_timestamp_ms =
Niels Möller23775882018-08-16 10:24:12 +02001227 1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency;
Sergey Silkin96f2c972018-09-05 21:07:17 +02001228 framerate_controller_[spatial_idx].AddFrame(frame_timestamp_ms);
Sergey Silkind902d582018-05-18 17:31:19 +02001229 }
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001230 }
1231}
1232
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001233int VP9EncoderImpl::RegisterEncodeCompleteCallback(
1234 EncodedImageCallback* callback) {
1235 encoded_complete_callback_ = callback;
1236 return WEBRTC_VIDEO_CODEC_OK;
1237}
1238
Erik Språng727d1642018-11-07 16:54:15 +01001239VideoEncoder::EncoderInfo VP9EncoderImpl::GetEncoderInfo() const {
1240 EncoderInfo info;
1241 info.supports_native_handle = false;
1242 info.implementation_name = "libvpx";
1243 info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
Erik Språngd3438aa2018-11-08 16:56:43 +01001244 info.has_trusted_rate_controller = trusted_rate_controller_;
Erik Språng727d1642018-11-07 16:54:15 +01001245 return info;
Peter Boströmb7d9a972015-12-18 16:01:11 +01001246}
1247
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001248VP9DecoderImpl::VP9DecoderImpl()
sprang3958ed82017-08-17 08:12:10 -07001249 : decode_complete_callback_(nullptr),
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001250 inited_(false),
sprang3958ed82017-08-17 08:12:10 -07001251 decoder_(nullptr),
philipel9d7d75b2018-04-04 13:03:01 +02001252 key_frame_required_(true) {}
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001253
1254VP9DecoderImpl::~VP9DecoderImpl() {
1255 inited_ = true; // in order to do the actual release
1256 Release();
Henrik Boström9695d852015-05-06 10:42:15 +02001257 int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse();
1258 if (num_buffers_in_use > 0) {
1259 // The frame buffers are reference counted and frames are exposed after
1260 // decoding. There may be valid usage cases where previous frames are still
1261 // referenced after ~VP9DecoderImpl that is not a leak.
Mirko Bonadei675513b2017-11-09 11:09:25 +01001262 RTC_LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still "
1263 << "referenced during ~VP9DecoderImpl.";
Henrik Boström9695d852015-05-06 10:42:15 +02001264 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001265}
1266
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001267int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001268 int ret_val = Release();
1269 if (ret_val < 0) {
1270 return ret_val;
1271 }
Sergey Silkinb674cd12018-10-09 10:59:06 +02001272
sprang3958ed82017-08-17 08:12:10 -07001273 if (decoder_ == nullptr) {
pbos@webrtc.orge728ee02014-12-17 13:43:55 +00001274 decoder_ = new vpx_codec_ctx_t;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001275 }
philipelcce46fc2015-12-21 03:04:49 -08001276 vpx_codec_dec_cfg_t cfg;
Sergey Silkinb674cd12018-10-09 10:59:06 +02001277 memset(&cfg, 0, sizeof(cfg));
1278
1279 // We want to use multithreading when decoding high resolution videos. But,
1280 // since we don't know resolution of input stream at this stage, we always
1281 // enable it.
1282 cfg.threads = std::min(number_of_cores, kMaxNumTiles4kVideo);
1283
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001284 vpx_codec_flags_t flags = 0;
1285 if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
1286 return WEBRTC_VIDEO_CODEC_MEMORY;
1287 }
Henrik Boström9695d852015-05-06 10:42:15 +02001288
1289 if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) {
1290 return WEBRTC_VIDEO_CODEC_MEMORY;
1291 }
1292
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001293 inited_ = true;
1294 // Always start with a complete key frame.
1295 key_frame_required_ = true;
1296 return WEBRTC_VIDEO_CODEC_OK;
1297}
1298
1299int VP9DecoderImpl::Decode(const EncodedImage& input_image,
1300 bool missing_frames,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001301 const CodecSpecificInfo* codec_specific_info,
1302 int64_t /*render_time_ms*/) {
1303 if (!inited_) {
1304 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1305 }
sprang3958ed82017-08-17 08:12:10 -07001306 if (decode_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001307 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1308 }
1309 // Always start with a complete key frame.
1310 if (key_frame_required_) {
Peter Boström49e196a2015-10-23 15:58:18 +02001311 if (input_image._frameType != kVideoFrameKey)
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001312 return WEBRTC_VIDEO_CODEC_ERROR;
1313 // We have a key frame - is it complete?
1314 if (input_image._completeFrame) {
1315 key_frame_required_ = false;
1316 } else {
1317 return WEBRTC_VIDEO_CODEC_ERROR;
1318 }
1319 }
sprang3958ed82017-08-17 08:12:10 -07001320 vpx_codec_iter_t iter = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001321 vpx_image_t* img;
1322 uint8_t* buffer = input_image._buffer;
1323 if (input_image._length == 0) {
sprang3958ed82017-08-17 08:12:10 -07001324 buffer = nullptr; // Triggers full frame concealment.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001325 }
Henrik Boström9695d852015-05-06 10:42:15 +02001326 // During decode libvpx may get and release buffers from |frame_buffer_pool_|.
1327 // In practice libvpx keeps a few (~3-4) buffers alive at a time.
philipelcce46fc2015-12-21 03:04:49 -08001328 if (vpx_codec_decode(decoder_, buffer,
1329 static_cast<unsigned int>(input_image._length), 0,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001330 VPX_DL_REALTIME)) {
1331 return WEBRTC_VIDEO_CODEC_ERROR;
1332 }
Henrik Boström9695d852015-05-06 10:42:15 +02001333 // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer.
1334 // It may be released by libvpx during future vpx_codec_decode or
1335 // vpx_codec_destroy calls.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001336 img = vpx_codec_get_frame(decoder_, &iter);
sakal7adadb12017-02-23 02:54:57 -08001337 int qp;
1338 vpx_codec_err_t vpx_ret =
1339 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
1340 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
Johannes Kron9973fa82018-11-07 14:39:26 +01001341 int ret = ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_,
Johannes Kron4749e4e2018-11-21 10:18:18 +01001342 qp, input_image.ColorSpace());
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001343 if (ret != 0) {
1344 return ret;
1345 }
1346 return WEBRTC_VIDEO_CODEC_OK;
1347}
1348
asapersson1490f7a2016-09-23 02:09:46 -07001349int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img,
1350 uint32_t timestamp,
sakal7adadb12017-02-23 02:54:57 -08001351 int64_t ntp_time_ms,
Johannes Kron9973fa82018-11-07 14:39:26 +01001352 int qp,
Johannes Kron4749e4e2018-11-21 10:18:18 +01001353 const ColorSpace* explicit_color_space) {
sprang3958ed82017-08-17 08:12:10 -07001354 if (img == nullptr) {
1355 // Decoder OK and nullptr image => No show frame.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001356 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1357 }
Henrik Boström9695d852015-05-06 10:42:15 +02001358
1359 // This buffer contains all of |img|'s image data, a reference counted
perkj14f41442015-11-30 22:15:45 -08001360 // Vp9FrameBuffer. (libvpx is done with the buffers after a few
Henrik Boström9695d852015-05-06 10:42:15 +02001361 // vpx_codec_decode calls or vpx_codec_destroy).
1362 Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer =
1363 static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv);
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001364
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -07001365 // The buffer can be used directly by the VideoFrame (without copy) by
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001366 // using a Wrapped*Buffer.
1367 rtc::scoped_refptr<VideoFrameBuffer> img_wrapped_buffer;
1368 switch (img->bit_depth) {
1369 case 8:
1370 img_wrapped_buffer = WrapI420Buffer(
philipelcce46fc2015-12-21 03:04:49 -08001371 img->d_w, img->d_h, img->planes[VPX_PLANE_Y],
1372 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U],
1373 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V],
1374 img->stride[VPX_PLANE_V],
Henrik Boström9695d852015-05-06 10:42:15 +02001375 // WrappedI420Buffer's mechanism for allowing the release of its frame
1376 // buffer is through a callback function. This is where we should
1377 // release |img_buffer|.
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001378 rtc::KeepRefUntilDone(img_buffer));
1379 break;
1380 case 10:
1381 img_wrapped_buffer = WrapI010Buffer(
1382 img->d_w, img->d_h,
1383 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]),
1384 img->stride[VPX_PLANE_Y] / 2,
1385 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]),
1386 img->stride[VPX_PLANE_U] / 2,
1387 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]),
1388 img->stride[VPX_PLANE_V] / 2, rtc::KeepRefUntilDone(img_buffer));
1389 break;
1390 default:
1391 RTC_NOTREACHED();
1392 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1393 }
nisseca6d5d12016-06-17 05:03:04 -07001394
Johannes Kron4749e4e2018-11-21 10:18:18 +01001395 auto builder = VideoFrame::Builder()
1396 .set_video_frame_buffer(img_wrapped_buffer)
1397 .set_timestamp_ms(0)
1398 .set_timestamp_rtp(timestamp)
1399 .set_ntp_time_ms(ntp_time_ms)
1400 .set_rotation(webrtc::kVideoRotation_0);
1401 if (explicit_color_space) {
1402 builder.set_color_space(explicit_color_space);
1403 } else {
1404 builder.set_color_space(
1405 ExtractVP9ColorSpace(img->cs, img->range, img->bit_depth));
1406 }
1407
1408 VideoFrame decoded_image = builder.build();
1409
Danil Chapovalov0040b662018-06-18 10:48:16 +02001410 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001411 return WEBRTC_VIDEO_CODEC_OK;
1412}
1413
1414int VP9DecoderImpl::RegisterDecodeCompleteCallback(
1415 DecodedImageCallback* callback) {
1416 decode_complete_callback_ = callback;
1417 return WEBRTC_VIDEO_CODEC_OK;
1418}
1419
1420int VP9DecoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001421 int ret_val = WEBRTC_VIDEO_CODEC_OK;
1422
sprang3958ed82017-08-17 08:12:10 -07001423 if (decoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +01001424 if (inited_) {
1425 // When a codec is destroyed libvpx will release any buffers of
1426 // |frame_buffer_pool_| it is currently using.
1427 if (vpx_codec_destroy(decoder_)) {
1428 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
1429 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001430 }
1431 delete decoder_;
sprang3958ed82017-08-17 08:12:10 -07001432 decoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001433 }
Henrik Boström9695d852015-05-06 10:42:15 +02001434 // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
1435 // still referenced externally are deleted once fully released, not returning
1436 // to the pool.
1437 frame_buffer_pool_.ClearPool();
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001438 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001439 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001440}
Peter Boströmb7d9a972015-12-18 16:01:11 +01001441
1442const char* VP9DecoderImpl::ImplementationName() const {
1443 return "libvpx";
1444}
1445
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001446} // namespace webrtc
Mirko Bonadei95adedb2018-11-19 09:52:37 +01001447
1448#endif // RTC_ENABLE_VP9