blob: ba0d7162a4ee4ed7f7126846a221067b4066c07c [file] [log] [blame]
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020012#include "modules/video_coding/codecs/vp9/vp9_impl.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000013
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020014#include <algorithm>
Sergey Silkind45b3452018-05-31 16:00:24 +020015#include <limits>
marpan@webrtc.org5b883172014-11-01 06:10:48 +000016#include <vector>
17
marpan@webrtc.org5b883172014-11-01 06:10:48 +000018#include "vpx/vp8cx.h"
19#include "vpx/vp8dx.h"
Yves Gerey665174f2018-06-19 15:03:05 +020020#include "vpx/vpx_decoder.h"
21#include "vpx/vpx_encoder.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000022
Karl Wiberg918f50c2018-07-05 11:40:33 +020023#include "absl/memory/memory.h"
Emircan Uysaler800787f2018-07-16 10:01:49 -070024#include "api/video/color_space.h"
Emircan Uysaler0823eec2018-07-13 17:10:00 -070025#include "api/video/i010_buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "common_video/include/video_frame_buffer.h"
27#include "common_video/libyuv/include/webrtc_libyuv.h"
Sergey Silkind902d582018-05-18 17:31:19 +020028#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Sergey Silkin86684962018-03-28 19:32:37 +020029#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "rtc_base/checks.h"
31#include "rtc_base/keep_ref_until_done.h"
32#include "rtc_base/logging.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020033#include "rtc_base/timeutils.h"
34#include "rtc_base/trace_event.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000035
36namespace webrtc {
37
Sergey Silkind902d582018-05-18 17:31:19 +020038namespace {
Marco6e89b252015-07-07 14:40:38 -070039// Only positive speeds, range for real-time coding currently is: 5 - 8.
40// Lower means slower/better quality, higher means fastest/lower quality.
41int GetCpuSpeed(int width, int height) {
Alex Glaznevfecb7c32016-03-31 14:23:27 -070042#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
Marco002f0d02015-12-17 09:49:31 -080043 return 8;
44#else
Marco6e89b252015-07-07 14:40:38 -070045 // For smaller resolutions, use lower speed setting (get some coding gain at
46 // the cost of increased encoding complexity).
47 if (width * height <= 352 * 288)
48 return 5;
49 else
50 return 7;
Marco002f0d02015-12-17 09:49:31 -080051#endif
Marco6e89b252015-07-07 14:40:38 -070052}
Emircan Uysaler715cc232018-07-23 17:50:43 -070053// Helper class for extracting VP9 colorspace.
54ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t,
55 vpx_color_range_t range_t,
56 unsigned int bit_depth) {
57 ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid;
58 ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid;
59 ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid;
60 switch (space_t) {
61 case VPX_CS_BT_601:
62 case VPX_CS_SMPTE_170:
63 primaries = ColorSpace::PrimaryID::kSMPTE170M;
64 transfer = ColorSpace::TransferID::kSMPTE170M;
65 matrix = ColorSpace::MatrixID::kSMPTE170M;
66 break;
67 case VPX_CS_SMPTE_240:
68 primaries = ColorSpace::PrimaryID::kSMPTE240M;
69 transfer = ColorSpace::TransferID::kSMPTE240M;
70 matrix = ColorSpace::MatrixID::kSMPTE240M;
71 break;
72 case VPX_CS_BT_709:
73 primaries = ColorSpace::PrimaryID::kBT709;
74 transfer = ColorSpace::TransferID::kBT709;
75 matrix = ColorSpace::MatrixID::kBT709;
76 break;
77 case VPX_CS_BT_2020:
78 primaries = ColorSpace::PrimaryID::kBT2020;
79 switch (bit_depth) {
80 case 8:
81 transfer = ColorSpace::TransferID::kBT709;
82 break;
83 case 10:
84 transfer = ColorSpace::TransferID::kBT2020_10;
85 break;
86 default:
87 RTC_NOTREACHED();
88 break;
89 }
90 matrix = ColorSpace::MatrixID::kBT2020_NCL;
91 break;
92 case VPX_CS_SRGB:
93 primaries = ColorSpace::PrimaryID::kBT709;
94 transfer = ColorSpace::TransferID::kIEC61966_2_1;
95 matrix = ColorSpace::MatrixID::kBT709;
96 break;
97 default:
98 break;
99 }
100
101 ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
102 switch (range_t) {
103 case VPX_CR_STUDIO_RANGE:
104 range = ColorSpace::RangeID::kLimited;
105 break;
106 case VPX_CR_FULL_RANGE:
107 range = ColorSpace::RangeID::kFull;
108 break;
109 default:
110 break;
111 }
112 return ColorSpace(primaries, transfer, matrix, range);
113}
114} // namespace
Marco6e89b252015-07-07 14:40:38 -0700115
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700116std::vector<SdpVideoFormat> SupportedVP9Codecs() {
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700117 // TODO(emircan): Add Profile 2 support after fixing browser_tests.
118 std::vector<SdpVideoFormat> supported_formats{SdpVideoFormat(
119 cricket::kVp9CodecName,
120 {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})};
121 return supported_formats;
Peter Boström12996152016-05-14 02:03:18 +0200122}
123
Magnus Jedvert46a27652017-11-13 14:10:02 +0100124std::unique_ptr<VP9Encoder> VP9Encoder::Create() {
Karl Wiberg918f50c2018-07-05 11:40:33 +0200125 return absl::make_unique<VP9EncoderImpl>(cricket::VideoCodec());
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700126}
127
128std::unique_ptr<VP9Encoder> VP9Encoder::Create(
129 const cricket::VideoCodec& codec) {
Karl Wiberg918f50c2018-07-05 11:40:33 +0200130 return absl::make_unique<VP9EncoderImpl>(codec);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000131}
132
asaperssona9455ab2015-07-31 06:10:09 -0700133void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
134 void* user_data) {
philipelcce46fc2015-12-21 03:04:49 -0800135 VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data);
asaperssona9455ab2015-07-31 06:10:09 -0700136 enc->GetEncodedLayerFrame(pkt);
137}
138
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700139VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000140 : encoded_image_(),
sprang3958ed82017-08-17 08:12:10 -0700141 encoded_complete_callback_(nullptr),
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700142 profile_(
143 ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000144 inited_(false),
145 timestamp_(0),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000146 cpu_speed_(3),
147 rc_max_intra_target_(0),
sprang3958ed82017-08-17 08:12:10 -0700148 encoder_(nullptr),
149 config_(nullptr),
150 raw_(nullptr),
151 input_image_(nullptr),
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200152 force_key_frame_(true),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200153 pics_since_key_(0),
asaperssona9455ab2015-07-31 06:10:09 -0700154 num_temporal_layers_(0),
philipelcfc319b2015-11-10 07:17:23 -0800155 num_spatial_layers_(0),
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200156 is_svc_(false),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200157 inter_layer_pred_(InterLayerPredMode::kOn),
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200158 is_flexible_mode_(false) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000159 memset(&codec_, 0, sizeof(codec_));
johannkoenig8225c402017-01-26 13:23:44 -0800160 memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000161}
162
163VP9EncoderImpl::~VP9EncoderImpl() {
164 Release();
165}
166
167int VP9EncoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100168 int ret_val = WEBRTC_VIDEO_CODEC_OK;
169
sprang3958ed82017-08-17 08:12:10 -0700170 if (encoded_image_._buffer != nullptr) {
philipelcce46fc2015-12-21 03:04:49 -0800171 delete[] encoded_image_._buffer;
sprang3958ed82017-08-17 08:12:10 -0700172 encoded_image_._buffer = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000173 }
sprang3958ed82017-08-17 08:12:10 -0700174 if (encoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +0100175 if (inited_) {
176 if (vpx_codec_destroy(encoder_)) {
177 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
178 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000179 }
180 delete encoder_;
sprang3958ed82017-08-17 08:12:10 -0700181 encoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000182 }
sprang3958ed82017-08-17 08:12:10 -0700183 if (config_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000184 delete config_;
sprang3958ed82017-08-17 08:12:10 -0700185 config_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000186 }
sprang3958ed82017-08-17 08:12:10 -0700187 if (raw_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000188 vpx_img_free(raw_);
sprang3958ed82017-08-17 08:12:10 -0700189 raw_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000190 }
191 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100192 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000193}
194
sprangce4aef12015-11-02 07:23:20 -0800195bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const {
196 // We check target_bitrate_bps of the 0th layer to see if the spatial layers
197 // (i.e. bitrates) were explicitly configured.
Sergey Silkin13e74342018-03-02 12:28:00 +0100198 return num_spatial_layers_ > 1 && codec_.spatialLayers[0].targetBitrate > 0;
sprangce4aef12015-11-02 07:23:20 -0800199}
200
Erik Språng566124a2018-04-23 12:32:22 +0200201bool VP9EncoderImpl::SetSvcRates(
202 const VideoBitrateAllocation& bitrate_allocation) {
Sergey Silkin86684962018-03-28 19:32:37 +0200203 config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
Sergey Silkin86684962018-03-28 19:32:37 +0200204
sprangce4aef12015-11-02 07:23:20 -0800205 if (ExplicitlyConfiguredSpatialLayers()) {
Sergey Silkin86684962018-03-28 19:32:37 +0200206 for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200207 const bool was_layer_enabled = (config_->ss_target_bitrate[sl_idx] > 0);
Sergey Silkin86684962018-03-28 19:32:37 +0200208 config_->ss_target_bitrate[sl_idx] =
209 bitrate_allocation.GetSpatialLayerSum(sl_idx) / 1000;
210
211 for (size_t tl_idx = 0; tl_idx < num_temporal_layers_; ++tl_idx) {
212 config_->layer_target_bitrate[sl_idx * num_temporal_layers_ + tl_idx] =
213 bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
214 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200215
216 const bool is_layer_enabled = (config_->ss_target_bitrate[sl_idx] > 0);
217 if (is_layer_enabled && !was_layer_enabled) {
218 if (inter_layer_pred_ == InterLayerPredMode::kOff ||
219 inter_layer_pred_ == InterLayerPredMode::kOnKeyPic) {
220 // TODO(wemb:1526): remove key frame request when issue is fixed.
221 force_key_frame_ = true;
222 }
223 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200224
225 if (!was_layer_enabled) {
226 // Reset frame rate controller if layer is resumed after pause.
227 framerate_controller_[sl_idx].Reset();
228 }
229
230 framerate_controller_[sl_idx].SetTargetRate(
231 codec_.spatialLayers[sl_idx].maxFramerate);
sprangce4aef12015-11-02 07:23:20 -0800232 }
233 } else {
234 float rate_ratio[VPX_MAX_LAYERS] = {0};
235 float total = 0;
Sergey Silkin908689d2018-08-20 09:28:45 +0200236 for (int i = 0; i < num_spatial_layers_; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800237 if (svc_params_.scaling_factor_num[i] <= 0 ||
238 svc_params_.scaling_factor_den[i] <= 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100239 RTC_LOG(LS_ERROR) << "Scaling factors not specified!";
sprangce4aef12015-11-02 07:23:20 -0800240 return false;
241 }
Yves Gerey665174f2018-06-19 15:03:05 +0200242 rate_ratio[i] = static_cast<float>(svc_params_.scaling_factor_num[i]) /
243 svc_params_.scaling_factor_den[i];
sprangce4aef12015-11-02 07:23:20 -0800244 total += rate_ratio[i];
245 }
246
Sergey Silkin908689d2018-08-20 09:28:45 +0200247 for (int i = 0; i < num_spatial_layers_; ++i) {
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200248 RTC_CHECK_GT(total, 0);
sprangce4aef12015-11-02 07:23:20 -0800249 config_->ss_target_bitrate[i] = static_cast<unsigned int>(
250 config_->rc_target_bitrate * rate_ratio[i] / total);
251 if (num_temporal_layers_ == 1) {
252 config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i];
253 } else if (num_temporal_layers_ == 2) {
254 config_->layer_target_bitrate[i * num_temporal_layers_] =
255 config_->ss_target_bitrate[i] * 2 / 3;
256 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
257 config_->ss_target_bitrate[i];
258 } else if (num_temporal_layers_ == 3) {
259 config_->layer_target_bitrate[i * num_temporal_layers_] =
260 config_->ss_target_bitrate[i] / 2;
261 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
262 config_->layer_target_bitrate[i * num_temporal_layers_] +
263 (config_->ss_target_bitrate[i] / 4);
264 config_->layer_target_bitrate[i * num_temporal_layers_ + 2] =
265 config_->ss_target_bitrate[i];
266 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100267 RTC_LOG(LS_ERROR) << "Unsupported number of temporal layers: "
268 << num_temporal_layers_;
sprangce4aef12015-11-02 07:23:20 -0800269 return false;
270 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200271
272 framerate_controller_[i].SetTargetRate(codec_.maxFramerate);
asaperssona9455ab2015-07-31 06:10:09 -0700273 }
274 }
Sergey Silkin908689d2018-08-20 09:28:45 +0200275
276 num_active_spatial_layers_ = 0;
277 for (int i = 0; i < num_spatial_layers_; ++i) {
278 if (config_->ss_target_bitrate[i] > 0) {
279 ++num_active_spatial_layers_;
280 }
281 }
282 RTC_DCHECK_GT(num_active_spatial_layers_, 0);
283
asaperssona9455ab2015-07-31 06:10:09 -0700284 return true;
285}
286
Erik Språng08127a92016-11-16 16:41:30 +0100287int VP9EncoderImpl::SetRateAllocation(
Erik Språng566124a2018-04-23 12:32:22 +0200288 const VideoBitrateAllocation& bitrate_allocation,
Erik Språng08127a92016-11-16 16:41:30 +0100289 uint32_t frame_rate) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000290 if (!inited_) {
291 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
292 }
293 if (encoder_->err) {
294 return WEBRTC_VIDEO_CODEC_ERROR;
295 }
Erik Språng08127a92016-11-16 16:41:30 +0100296 if (frame_rate < 1) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000297 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
298 }
299 // Update bit rate
Erik Språng08127a92016-11-16 16:41:30 +0100300 if (codec_.maxBitrate > 0 &&
301 bitrate_allocation.get_sum_kbps() > codec_.maxBitrate) {
302 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000303 }
Erik Språng08127a92016-11-16 16:41:30 +0100304
Erik Språng08127a92016-11-16 16:41:30 +0100305 codec_.maxFramerate = frame_rate;
asaperssona9455ab2015-07-31 06:10:09 -0700306
Sergey Silkin86684962018-03-28 19:32:37 +0200307 if (!SetSvcRates(bitrate_allocation)) {
asaperssona9455ab2015-07-31 06:10:09 -0700308 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
309 }
310
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000311 // Update encoder context
312 if (vpx_codec_enc_config_set(encoder_, config_)) {
313 return WEBRTC_VIDEO_CODEC_ERROR;
314 }
315 return WEBRTC_VIDEO_CODEC_OK;
316}
317
318int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
319 int number_of_cores,
pkasting@chromium.org4591fbd2014-11-20 22:28:14 +0000320 size_t /*max_payload_size*/) {
sprang3958ed82017-08-17 08:12:10 -0700321 if (inst == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000322 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
323 }
324 if (inst->maxFramerate < 1) {
325 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
326 }
327 // Allow zero to represent an unspecified maxBitRate
328 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
329 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
330 }
331 if (inst->width < 1 || inst->height < 1) {
332 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
333 }
334 if (number_of_cores < 1) {
335 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
336 }
hta257dc392016-10-25 09:05:06 -0700337 if (inst->VP9().numberOfTemporalLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700338 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
339 }
ilnik2a8c2f52017-02-15 02:23:28 -0800340 // libvpx probably does not support more than 3 spatial layers.
341 if (inst->VP9().numberOfSpatialLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700342 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
343 }
philipelcfc319b2015-11-10 07:17:23 -0800344
asapersson86956de2016-01-26 01:05:20 -0800345 int ret_val = Release();
346 if (ret_val < 0) {
347 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000348 }
sprang3958ed82017-08-17 08:12:10 -0700349 if (encoder_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000350 encoder_ = new vpx_codec_ctx_t;
351 }
sprang3958ed82017-08-17 08:12:10 -0700352 if (config_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000353 config_ = new vpx_codec_enc_cfg_t;
354 }
355 timestamp_ = 0;
356 if (&codec_ != inst) {
357 codec_ = *inst;
358 }
asaperssona9455ab2015-07-31 06:10:09 -0700359
hta257dc392016-10-25 09:05:06 -0700360 num_spatial_layers_ = inst->VP9().numberOfSpatialLayers;
Niels Möller65fb4042018-04-25 14:46:06 +0200361 RTC_DCHECK_GT(num_spatial_layers_, 0);
hta257dc392016-10-25 09:05:06 -0700362 num_temporal_layers_ = inst->VP9().numberOfTemporalLayers;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200363 if (num_temporal_layers_ == 0) {
asaperssona9455ab2015-07-31 06:10:09 -0700364 num_temporal_layers_ = 1;
Sergey Silkin042661b2018-09-10 10:39:35 +0000365 }
Sergey Silkinae9e1882018-09-05 21:07:17 +0200366
Sergey Silkin96f2c972018-09-05 21:07:17 +0200367 framerate_controller_ = std::vector<FramerateController>(
368 num_spatial_layers_, FramerateController(codec_.maxFramerate));
369
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200370 is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200371
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000372 // Allocate memory for encoded image
sprang3958ed82017-08-17 08:12:10 -0700373 if (encoded_image_._buffer != nullptr) {
philipelcce46fc2015-12-21 03:04:49 -0800374 delete[] encoded_image_._buffer;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000375 }
nisseeb44b392017-04-28 07:18:05 -0700376 encoded_image_._size =
377 CalcBufferSize(VideoType::kI420, codec_.width, codec_.height);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000378 encoded_image_._buffer = new uint8_t[encoded_image_._size];
379 encoded_image_._completeFrame = true;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000380 // Populate encoder configuration with default values.
381 if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) {
382 return WEBRTC_VIDEO_CODEC_ERROR;
383 }
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700384
385 vpx_img_fmt img_fmt = VPX_IMG_FMT_NONE;
386 unsigned int bits_for_storage = 8;
387 switch (profile_) {
388 case VP9Profile::kProfile0:
389 img_fmt = VPX_IMG_FMT_I420;
390 bits_for_storage = 8;
391 config_->g_bit_depth = VPX_BITS_8;
392 config_->g_profile = 0;
393 config_->g_input_bit_depth = 8;
394 break;
395 case VP9Profile::kProfile2:
396 img_fmt = VPX_IMG_FMT_I42016;
397 bits_for_storage = 16;
398 config_->g_bit_depth = VPX_BITS_10;
399 config_->g_profile = 2;
400 config_->g_input_bit_depth = 10;
401 break;
402 }
403
404 // Creating a wrapper to the image - setting image data to nullptr. Actual
405 // pointer will be set in encode. Setting align to 1, as it is meaningless
406 // (actual memory is not allocated).
407 raw_ =
408 vpx_img_wrap(nullptr, img_fmt, codec_.width, codec_.height, 1, nullptr);
409 raw_->bit_depth = bits_for_storage;
410
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000411 config_->g_w = codec_.width;
412 config_->g_h = codec_.height;
413 config_->rc_target_bitrate = inst->startBitrate; // in kbit/s
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200414 config_->g_error_resilient = is_svc_ ? VPX_ERROR_RESILIENT_DEFAULT : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000415 // Setting the time base of the codec.
416 config_->g_timebase.num = 1;
417 config_->g_timebase.den = 90000;
418 config_->g_lag_in_frames = 0; // 0- no frame lagging
419 config_->g_threads = 1;
420 // Rate control settings.
hta257dc392016-10-25 09:05:06 -0700421 config_->rc_dropframe_thresh = inst->VP9().frameDroppingOn ? 30 : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000422 config_->rc_end_usage = VPX_CBR;
423 config_->g_pass = VPX_RC_ONE_PASS;
424 config_->rc_min_quantizer = 2;
marpan@webrtc.orgdc8a9da2015-01-27 23:08:24 +0000425 config_->rc_max_quantizer = 52;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000426 config_->rc_undershoot_pct = 50;
427 config_->rc_overshoot_pct = 50;
428 config_->rc_buf_initial_sz = 500;
429 config_->rc_buf_optimal_sz = 600;
430 config_->rc_buf_sz = 1000;
431 // Set the maximum target size of any key-frame.
432 rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz);
hta257dc392016-10-25 09:05:06 -0700433 if (inst->VP9().keyFrameInterval > 0) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000434 config_->kf_mode = VPX_KF_AUTO;
hta257dc392016-10-25 09:05:06 -0700435 config_->kf_max_dist = inst->VP9().keyFrameInterval;
Åsa Perssonff24c042015-12-04 10:58:08 +0100436 // Needs to be set (in svc mode) to get correct periodic key frame interval
437 // (will have no effect in non-svc).
438 config_->kf_min_dist = config_->kf_max_dist;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000439 } else {
440 config_->kf_mode = VPX_KF_DISABLED;
441 }
hta257dc392016-10-25 09:05:06 -0700442 config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000443 // Determine number of threads based on the image size and #cores.
philipelcce46fc2015-12-21 03:04:49 -0800444 config_->g_threads =
445 NumberOfThreads(config_->g_w, config_->g_h, number_of_cores);
asaperssona9455ab2015-07-31 06:10:09 -0700446
Marco6e89b252015-07-07 14:40:38 -0700447 cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h);
asaperssona9455ab2015-07-31 06:10:09 -0700448
hta257dc392016-10-25 09:05:06 -0700449 is_flexible_mode_ = inst->VP9().flexibleMode;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200450
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200451 if (num_temporal_layers_ == 1) {
asaperssona9455ab2015-07-31 06:10:09 -0700452 gof_.SetGofInfoVP9(kTemporalStructureMode1);
453 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING;
454 config_->ts_number_layers = 1;
455 config_->ts_rate_decimator[0] = 1;
456 config_->ts_periodicity = 1;
457 config_->ts_layer_id[0] = 0;
458 } else if (num_temporal_layers_ == 2) {
459 gof_.SetGofInfoVP9(kTemporalStructureMode2);
460 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
461 config_->ts_number_layers = 2;
462 config_->ts_rate_decimator[0] = 2;
463 config_->ts_rate_decimator[1] = 1;
464 config_->ts_periodicity = 2;
465 config_->ts_layer_id[0] = 0;
466 config_->ts_layer_id[1] = 1;
467 } else if (num_temporal_layers_ == 3) {
468 gof_.SetGofInfoVP9(kTemporalStructureMode3);
469 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212;
470 config_->ts_number_layers = 3;
471 config_->ts_rate_decimator[0] = 4;
472 config_->ts_rate_decimator[1] = 2;
473 config_->ts_rate_decimator[2] = 1;
474 config_->ts_periodicity = 4;
475 config_->ts_layer_id[0] = 0;
476 config_->ts_layer_id[1] = 2;
477 config_->ts_layer_id[2] = 1;
478 config_->ts_layer_id[3] = 2;
479 } else {
480 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
481 }
482
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200483 inter_layer_pred_ = inst->VP9().interLayerPred;
484
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200485 ref_buf_.clear();
486
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000487 return InitAndSetControlSettings(inst);
488}
489
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000490int VP9EncoderImpl::NumberOfThreads(int width,
491 int height,
492 int number_of_cores) {
493 // Keep the number of encoder threads equal to the possible number of column
494 // tiles, which is (1, 2, 4, 8). See comments below for VP9E_SET_TILE_COLUMNS.
495 if (width * height >= 1280 * 720 && number_of_cores > 4) {
496 return 4;
jianj23173a32017-07-12 16:11:09 -0700497 } else if (width * height >= 640 * 360 && number_of_cores > 2) {
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000498 return 2;
499 } else {
Yves Gerey665174f2018-06-19 15:03:05 +0200500// Use 2 threads for low res on ARM.
Jerome Jiang831af372017-12-05 10:44:35 -0800501#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
502 defined(WEBRTC_ANDROID)
503 if (width * height >= 320 * 180 && number_of_cores > 2) {
504 return 2;
505 }
506#endif
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000507 // 1 thread less than VGA.
508 return 1;
509 }
510}
511
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000512int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) {
Åsa Perssonff24c042015-12-04 10:58:08 +0100513 // Set QP-min/max per spatial and temporal layer.
514 int tot_num_layers = num_spatial_layers_ * num_temporal_layers_;
515 for (int i = 0; i < tot_num_layers; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800516 svc_params_.max_quantizers[i] = config_->rc_max_quantizer;
517 svc_params_.min_quantizers[i] = config_->rc_min_quantizer;
Åsa Perssonff24c042015-12-04 10:58:08 +0100518 }
asaperssona9455ab2015-07-31 06:10:09 -0700519 config_->ss_number_layers = num_spatial_layers_;
sprangce4aef12015-11-02 07:23:20 -0800520 if (ExplicitlyConfiguredSpatialLayers()) {
521 for (int i = 0; i < num_spatial_layers_; ++i) {
522 const auto& layer = codec_.spatialLayers[i];
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200523 RTC_CHECK_GT(layer.width, 0);
Sergey Silkin13e74342018-03-02 12:28:00 +0100524 const int scale_factor = codec_.width / layer.width;
525 RTC_DCHECK_GT(scale_factor, 0);
526
527 // Ensure scaler factor is integer.
528 if (scale_factor * layer.width != codec_.width) {
529 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
530 }
531
532 // Ensure scale factor is the same in both dimensions.
533 if (scale_factor * layer.height != codec_.height) {
534 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
535 }
536
537 // Ensure scale factor is power of two.
538 const bool is_pow_of_two = (scale_factor & (scale_factor - 1)) == 0;
539 if (!is_pow_of_two) {
540 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
541 }
542
543 svc_params_.scaling_factor_num[i] = 1;
544 svc_params_.scaling_factor_den[i] = scale_factor;
Sergey Silkin96f2c972018-09-05 21:07:17 +0200545
546 RTC_DCHECK_GT(codec_.spatialLayers[i].maxFramerate, 0);
547 RTC_DCHECK_LE(codec_.spatialLayers[i].maxFramerate, codec_.maxFramerate);
548 if (i > 0) {
549 // Frame rate of high spatial layer is supposed to be equal or higher
550 // than frame rate of low spatial layer.
551 RTC_DCHECK_GE(codec_.spatialLayers[i].maxFramerate,
552 codec_.spatialLayers[i - 1].maxFramerate);
553 }
sprangce4aef12015-11-02 07:23:20 -0800554 }
555 } else {
556 int scaling_factor_num = 256;
557 for (int i = num_spatial_layers_ - 1; i >= 0; --i) {
sprangce4aef12015-11-02 07:23:20 -0800558 // 1:2 scaling in each dimension.
johannkoenig8225c402017-01-26 13:23:44 -0800559 svc_params_.scaling_factor_num[i] = scaling_factor_num;
560 svc_params_.scaling_factor_den[i] = 256;
sprangce4aef12015-11-02 07:23:20 -0800561 }
asaperssona9455ab2015-07-31 06:10:09 -0700562 }
563
Sergey Silkin86684962018-03-28 19:32:37 +0200564 SvcRateAllocator init_allocator(codec_);
Erik Språng566124a2018-04-23 12:32:22 +0200565 VideoBitrateAllocation allocation = init_allocator.GetAllocation(
Sergey Silkin86684962018-03-28 19:32:37 +0200566 inst->startBitrate * 1000, inst->maxFramerate);
567 if (!SetSvcRates(allocation)) {
asaperssona9455ab2015-07-31 06:10:09 -0700568 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
569 }
570
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700571 const vpx_codec_err_t rv = vpx_codec_enc_init(
572 encoder_, vpx_codec_vp9_cx(), config_,
573 config_->g_bit_depth == VPX_BITS_8 ? 0 : VPX_CODEC_USE_HIGHBITDEPTH);
574 if (rv != VPX_CODEC_OK) {
575 RTC_LOG(LS_ERROR) << "Init error: " << vpx_codec_err_to_string(rv);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000576 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
577 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000578 vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_);
579 vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT,
580 rc_max_intra_target_);
581 vpx_codec_control(encoder_, VP9E_SET_AQ_MODE,
hta257dc392016-10-25 09:05:06 -0700582 inst->VP9().adaptiveQpMode ? 3 : 0);
asaperssona9455ab2015-07-31 06:10:09 -0700583
jianj822e5932017-07-12 16:09:58 -0700584 vpx_codec_control(encoder_, VP9E_SET_FRAME_PARALLEL_DECODING, 0);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200585
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200586 if (is_svc_) {
587 vpx_codec_control(encoder_, VP9E_SET_SVC, 1);
588 vpx_codec_control(encoder_, VP9E_SET_SVC_PARAMETERS, &svc_params_);
asaperssona9455ab2015-07-31 06:10:09 -0700589 }
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200590
591 if (num_spatial_layers_ > 1) {
592 switch (inter_layer_pred_) {
593 case InterLayerPredMode::kOn:
594 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 0);
595 break;
596 case InterLayerPredMode::kOff:
597 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 1);
598 break;
599 case InterLayerPredMode::kOnKeyPic:
600 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 2);
601 break;
602 default:
603 RTC_NOTREACHED();
604 }
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200605
Sergey Silkinee203362018-05-30 11:34:08 +0200606 // Configure encoder to drop entire superframe whenever it needs to drop
607 // a layer. This mode is prefered over per-layer dropping which causes
608 // quality flickering and is not compatible with RTP non-flexible mode.
609 vpx_svc_frame_drop_t svc_drop_frame;
610 memset(&svc_drop_frame, 0, sizeof(svc_drop_frame));
611 svc_drop_frame.framedrop_mode = FULL_SUPERFRAME_DROP;
Sergey Silkind45b3452018-05-31 16:00:24 +0200612 svc_drop_frame.max_consec_drop = std::numeric_limits<int>::max();
Sergey Silkinee203362018-05-30 11:34:08 +0200613 for (size_t i = 0; i < num_spatial_layers_; ++i) {
614 svc_drop_frame.framedrop_thresh[i] = config_->rc_dropframe_thresh;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200615 }
Sergey Silkinee203362018-05-30 11:34:08 +0200616 vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200617 }
618
asaperssona9455ab2015-07-31 06:10:09 -0700619 // Register callback for getting each spatial layer.
620 vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = {
philipelcce46fc2015-12-21 03:04:49 -0800621 VP9EncoderImpl::EncoderOutputCodedPacketCallback,
622 reinterpret_cast<void*>(this)};
623 vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK,
624 reinterpret_cast<void*>(&cbp));
asaperssona9455ab2015-07-31 06:10:09 -0700625
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000626 // Control function to set the number of column tiles in encoding a frame, in
627 // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns.
628 // The number tile columns will be capped by the encoder based on image size
629 // (minimum width of tile column is 256 pixels, maximum is 4096).
630 vpx_codec_control(encoder_, VP9E_SET_TILE_COLUMNS, (config_->g_threads >> 1));
jianjcb5d1152017-03-28 23:56:08 -0700631
632 // Turn on row-based multithreading.
633 vpx_codec_control(encoder_, VP9E_SET_ROW_MT, 1);
jianj6bf57e32017-06-05 13:43:49 -0700634
Alex Glaznevfecb7c32016-03-31 14:23:27 -0700635#if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
Yves Gerey665174f2018-06-19 15:03:05 +0200636 !defined(ANDROID)
jianj6bf57e32017-06-05 13:43:49 -0700637 // Do not enable the denoiser on ARM since optimization is pending.
638 // Denoiser is on by default on other platforms.
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000639 vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY,
hta257dc392016-10-25 09:05:06 -0700640 inst->VP9().denoisingOn ? 1 : 0);
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000641#endif
jianj6bf57e32017-06-05 13:43:49 -0700642
Niels Möllere3cf3d02018-06-13 11:52:16 +0200643 if (codec_.mode == VideoCodecMode::kScreensharing) {
ivica242d6382015-09-04 06:13:23 -0700644 // Adjust internal parameters to screen content.
645 vpx_codec_control(encoder_, VP9E_SET_TUNE_CONTENT, 1);
ivica242d6382015-09-04 06:13:23 -0700646 }
Marco2520e722015-09-16 14:05:00 -0700647 // Enable encoder skip of static/low content blocks.
648 vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000649 inited_ = true;
650 return WEBRTC_VIDEO_CODEC_OK;
651}
652
653uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) {
654 // Set max to the optimal buffer level (normalized by target BR),
655 // and scaled by a scale_par.
656 // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps].
657 // This value is presented in percentage of perFrameBw:
658 // perFrameBw = targetBR[Kbps] * 1000 / framerate.
659 // The target in % is as follows:
660 float scale_par = 0.5;
661 uint32_t target_pct =
662 optimal_buffer_size * scale_par * codec_.maxFramerate / 10;
663 // Don't go below 3 times the per frame bandwidth.
664 const uint32_t min_intra_size = 300;
philipelcce46fc2015-12-21 03:04:49 -0800665 return (target_pct < min_intra_size) ? min_intra_size : target_pct;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000666}
667
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700668int VP9EncoderImpl::Encode(const VideoFrame& input_image,
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000669 const CodecSpecificInfo* codec_specific_info,
pbos22993e12015-10-19 02:39:06 -0700670 const std::vector<FrameType>* frame_types) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000671 if (!inited_) {
672 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
673 }
sprang3958ed82017-08-17 08:12:10 -0700674 if (encoded_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000675 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
676 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200677
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000678 // We only support one stream at the moment.
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200679 if (frame_types && !frame_types->empty()) {
680 if ((*frame_types)[0] == kVideoFrameKey) {
681 force_key_frame_ = true;
682 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000683 }
Sergey Silkind902d582018-05-18 17:31:19 +0200684
Niels Möllere3cf3d02018-06-13 11:52:16 +0200685 if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) {
Sergey Silkin96f2c972018-09-05 21:07:17 +0200686 // Skip encoding spatial layer frames if their target frame rate is lower
687 // than actual input frame rate.
688 vpx_svc_layer_id_t layer_id = {0};
689 const size_t gof_idx = (pics_since_key_ + 1) % gof_.num_frames_in_gof;
690 layer_id.temporal_layer_id = gof_.temporal_idx[gof_idx];
691
692 const uint32_t frame_timestamp_ms =
693 1000 * input_image.timestamp() / kVideoPayloadTypeFrequency;
694
695 for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) {
696 if (framerate_controller_[sl_idx].DropFrame(frame_timestamp_ms)) {
697 ++layer_id.spatial_layer_id;
698 } else {
699 break;
700 }
701 }
702
703 RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_);
704 if (layer_id.spatial_layer_id >= num_active_spatial_layers_) {
705 // Drop entire picture.
Sergey Silkind902d582018-05-18 17:31:19 +0200706 return WEBRTC_VIDEO_CODEC_OK;
707 }
Sergey Silkin96f2c972018-09-05 21:07:17 +0200708
709 vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id);
Sergey Silkind902d582018-05-18 17:31:19 +0200710 }
711
kwiberg352444f2016-11-28 15:58:53 -0800712 RTC_DCHECK_EQ(input_image.width(), raw_->d_w);
713 RTC_DCHECK_EQ(input_image.height(), raw_->d_h);
asaperssona9455ab2015-07-31 06:10:09 -0700714
715 // Set input image for use in the callback.
716 // This was necessary since you need some information from input_image.
717 // You can save only the necessary information (such as timestamp) instead of
718 // doing this.
719 input_image_ = &input_image;
720
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700721 // Keep reference to buffer until encode completes.
722 rtc::scoped_refptr<I420BufferInterface> i420_buffer;
723 rtc::scoped_refptr<I010BufferInterface> i010_buffer;
724 switch (profile_) {
725 case VP9Profile::kProfile0: {
726 i420_buffer = input_image.video_frame_buffer()->ToI420();
727 // Image in vpx_image_t format.
728 // Input image is const. VPX's raw image is not defined as const.
729 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
730 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU());
731 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV());
732 raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
733 raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU();
734 raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
735 break;
736 }
737 case VP9Profile::kProfile2: {
738 // We can inject kI010 frames directly for encode. All other formats
739 // should be converted to it.
740 switch (input_image.video_frame_buffer()->type()) {
741 case VideoFrameBuffer::Type::kI010: {
742 i010_buffer = input_image.video_frame_buffer()->GetI010();
743 break;
744 }
745 default: {
746 i010_buffer =
747 I010Buffer::Copy(*input_image.video_frame_buffer()->ToI420());
748 }
749 }
750 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(
751 reinterpret_cast<const uint8_t*>(i010_buffer->DataY()));
752 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(
753 reinterpret_cast<const uint8_t*>(i010_buffer->DataU()));
754 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(
755 reinterpret_cast<const uint8_t*>(i010_buffer->DataV()));
756 raw_->stride[VPX_PLANE_Y] = i010_buffer->StrideY() * 2;
757 raw_->stride[VPX_PLANE_U] = i010_buffer->StrideU() * 2;
758 raw_->stride[VPX_PLANE_V] = i010_buffer->StrideV() * 2;
759 break;
760 }
761 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000762
philipelcfc319b2015-11-10 07:17:23 -0800763 vpx_enc_frame_flags_t flags = 0;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200764 if (force_key_frame_) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000765 flags = VPX_EFLAG_FORCE_KF;
766 }
philipelcfc319b2015-11-10 07:17:23 -0800767
Sergey Silkin96f2c972018-09-05 21:07:17 +0200768 // TODO(ssilkin): Frame duration should be specified per spatial layer
769 // since their frame rate can be different. For now calculate frame duration
770 // based on target frame rate of the highest spatial layer, which frame rate
771 // is supposed to be equal or higher than frame rate of low spatial layers.
772 // Also, timestamp should represent actual time passed since previous frame
773 // (not 'expected' time). Then rate controller can drain buffer more
774 // accurately.
775 RTC_DCHECK_GE(framerate_controller_.size(), num_active_spatial_layers_);
776 uint32_t duration = static_cast<uint32_t>(
777 90000 /
778 framerate_controller_[num_active_spatial_layers_ - 1].GetTargetRate());
Emircan Uysaler0823eec2018-07-13 17:10:00 -0700779 const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_,
780 duration, flags, VPX_DL_REALTIME);
781 if (rv != VPX_CODEC_OK) {
782 RTC_LOG(LS_ERROR) << "Encoding error: " << vpx_codec_err_to_string(rv)
783 << "\n"
784 << "Details: " << vpx_codec_error(encoder_) << "\n"
785 << vpx_codec_error_detail(encoder_);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000786 return WEBRTC_VIDEO_CODEC_ERROR;
787 }
788 timestamp_ += duration;
asaperssona9455ab2015-07-31 06:10:09 -0700789
Sergey Silkinbc0f0d32018-04-24 21:29:14 +0200790 const bool end_of_picture = true;
791 DeliverBufferedFrame(end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200792
asaperssona9455ab2015-07-31 06:10:09 -0700793 return WEBRTC_VIDEO_CODEC_OK;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000794}
795
796void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
Niels Möllerd3b8c632018-08-27 15:33:42 +0200797 absl::optional<int>* spatial_idx,
philipelcce46fc2015-12-21 03:04:49 -0800798 const vpx_codec_cx_pkt& pkt,
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200799 uint32_t timestamp,
800 bool first_frame_in_picture) {
sprang3958ed82017-08-17 08:12:10 -0700801 RTC_CHECK(codec_specific != nullptr);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000802 codec_specific->codecType = kVideoCodecVP9;
perkj275afc52016-09-01 00:21:16 -0700803 codec_specific->codec_name = ImplementationName();
philipelcce46fc2015-12-21 03:04:49 -0800804 CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9);
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200805
806 vp9_info->first_frame_in_picture = first_frame_in_picture;
Sergey Silkin738b7e92018-08-23 16:37:27 +0200807 vp9_info->flexible_mode = is_flexible_mode_;
hta257dc392016-10-25 09:05:06 -0700808 vp9_info->ss_data_available =
Sergey Silkin738b7e92018-08-23 16:37:27 +0200809 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
asaperssona9455ab2015-07-31 06:10:09 -0700810
811 vpx_svc_layer_id_t layer_id = {0};
812 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
813
sprang3958ed82017-08-17 08:12:10 -0700814 RTC_CHECK_GT(num_temporal_layers_, 0);
“Michael23c5a992018-06-21 11:07:21 -0500815 RTC_CHECK_GT(num_active_spatial_layers_, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700816 if (num_temporal_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700817 RTC_CHECK_EQ(layer_id.temporal_layer_id, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700818 vp9_info->temporal_idx = kNoTemporalIdx;
819 } else {
820 vp9_info->temporal_idx = layer_id.temporal_layer_id;
821 }
“Michael23c5a992018-06-21 11:07:21 -0500822 if (num_active_spatial_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700823 RTC_CHECK_EQ(layer_id.spatial_layer_id, 0);
Niels Möllerd3b8c632018-08-27 15:33:42 +0200824 *spatial_idx = absl::nullopt;
asaperssona9455ab2015-07-31 06:10:09 -0700825 } else {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200826 *spatial_idx = layer_id.spatial_layer_id;
asaperssona9455ab2015-07-31 06:10:09 -0700827 }
828 if (layer_id.spatial_layer_id != 0) {
829 vp9_info->ss_data_available = false;
830 }
831
asaperssona9455ab2015-07-31 06:10:09 -0700832 // TODO(asapersson): this info has to be obtained from the encoder.
asaperssoncb50c962015-11-18 01:58:55 -0800833 vp9_info->temporal_up_switch = false;
asaperssona9455ab2015-07-31 06:10:09 -0700834
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200835 if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
836 pics_since_key_ = 0;
837 } else if (first_frame_in_picture) {
838 ++pics_since_key_;
asaperssona9455ab2015-07-31 06:10:09 -0700839 }
840
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200841 const bool is_key_pic = (pics_since_key_ == 0);
842 const bool is_inter_layer_pred_allowed =
843 (inter_layer_pred_ == InterLayerPredMode::kOn ||
844 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic));
845
846 // Always set inter_layer_predicted to true on high layer frame if inter-layer
847 // prediction (ILP) is allowed even if encoder didn't actually use it.
848 // Setting inter_layer_predicted to false would allow receiver to decode high
849 // layer frame without decoding low layer frame. If that would happen (e.g.
850 // if low layer frame is lost) then receiver won't be able to decode next high
851 // layer frame which uses ILP.
852 vp9_info->inter_layer_predicted =
853 first_frame_in_picture ? false : is_inter_layer_pred_allowed;
854
855 const bool is_last_layer =
“Michael23c5a992018-06-21 11:07:21 -0500856 (layer_id.spatial_layer_id + 1 == num_active_spatial_layers_);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200857 vp9_info->non_ref_for_inter_layer_pred =
858 is_last_layer ? true : !is_inter_layer_pred_allowed;
asapersson00ac85e2015-11-11 05:30:48 -0800859
ivica7f6a6fc2015-09-08 02:40:29 -0700860 // Always populate this, so that the packetizer can properly set the marker
861 // bit.
“Michael23c5a992018-06-21 11:07:21 -0500862 vp9_info->num_spatial_layers = num_active_spatial_layers_;
philipelcfc319b2015-11-10 07:17:23 -0800863
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200864 vp9_info->num_ref_pics = 0;
865 if (vp9_info->flexible_mode) {
866 vp9_info->gof_idx = kNoGofIdx;
867 FillReferenceIndices(pkt, pics_since_key_, vp9_info->inter_layer_predicted,
868 vp9_info);
869 } else {
870 vp9_info->gof_idx =
871 static_cast<uint8_t>(pics_since_key_ % gof_.num_frames_in_gof);
872 vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx];
873 vp9_info->num_ref_pics = gof_.num_ref_pics[vp9_info->gof_idx];
874 }
875
876 vp9_info->inter_pic_predicted = (!is_key_pic && vp9_info->num_ref_pics > 0);
877
asaperssona9455ab2015-07-31 06:10:09 -0700878 if (vp9_info->ss_data_available) {
asaperssona9455ab2015-07-31 06:10:09 -0700879 vp9_info->spatial_layer_resolution_present = true;
“Michael23c5a992018-06-21 11:07:21 -0500880 for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
Yves Gerey665174f2018-06-19 15:03:05 +0200881 vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -0800882 svc_params_.scaling_factor_den[i];
Yves Gerey665174f2018-06-19 15:03:05 +0200883 vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -0800884 svc_params_.scaling_factor_den[i];
asaperssona9455ab2015-07-31 06:10:09 -0700885 }
Sergey Silkin738b7e92018-08-23 16:37:27 +0200886 if (vp9_info->flexible_mode) {
887 vp9_info->gof.num_frames_in_gof = 0;
888 } else {
asaperssona9455ab2015-07-31 06:10:09 -0700889 vp9_info->gof.CopyGofInfoVP9(gof_);
890 }
891 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000892}
893
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200894void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
895 const size_t pic_num,
896 const bool inter_layer_predicted,
897 CodecSpecificInfoVP9* vp9_info) {
898 vpx_svc_layer_id_t layer_id = {0};
899 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
900
Sergey Silkin76be2952018-08-21 12:30:33 +0200901 const bool is_key_frame =
902 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200903
904 std::vector<RefFrameBuffer> ref_buf_list;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200905
Sergey Silkin76be2952018-08-21 12:30:33 +0200906 if (is_svc_) {
907 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
908 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200909
Sergey Silkin76be2952018-08-21 12:30:33 +0200910 if (enc_layer_conf.reference_last[layer_id.spatial_layer_id]) {
911 const size_t fb_idx =
912 enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id];
913 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +0200914 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
915 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
916 ref_buf_list.push_back(ref_buf_.at(fb_idx));
917 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200918 }
919
920 if (enc_layer_conf.reference_alt_ref[layer_id.spatial_layer_id]) {
921 const size_t fb_idx =
922 enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id];
923 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +0200924 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
925 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
926 ref_buf_list.push_back(ref_buf_.at(fb_idx));
927 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200928 }
929
930 if (enc_layer_conf.reference_golden[layer_id.spatial_layer_id]) {
931 const size_t fb_idx =
932 enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id];
933 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
Sergey Silkin82276592018-08-27 14:54:20 +0200934 if (std::find(ref_buf_list.begin(), ref_buf_list.end(),
935 ref_buf_.at(fb_idx)) == ref_buf_list.end()) {
936 ref_buf_list.push_back(ref_buf_.at(fb_idx));
937 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200938 }
939 } else if (!is_key_frame) {
940 RTC_DCHECK_EQ(num_spatial_layers_, 1);
941 RTC_DCHECK_EQ(num_temporal_layers_, 1);
942 // In non-SVC mode encoder doesn't provide reference list. Assume each frame
943 // refers previous one, which is stored in buffer 0.
944 ref_buf_list.push_back(ref_buf_.at(0));
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200945 }
946
947 size_t max_ref_temporal_layer_id = 0;
948
949 vp9_info->num_ref_pics = 0;
950 for (const RefFrameBuffer& ref_buf : ref_buf_list) {
951 RTC_DCHECK_LE(ref_buf.pic_num, pic_num);
952 if (ref_buf.pic_num < pic_num) {
953 if (inter_layer_pred_ != InterLayerPredMode::kOn) {
954 // RTP spec limits temporal prediction to the same spatial layer.
955 // It is safe to ignore this requirement if inter-layer prediction is
956 // enabled for all frames when all base frames are relayed to receiver.
957 RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id);
958 }
959 RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id);
960
961 const size_t p_diff = pic_num - ref_buf.pic_num;
962 RTC_DCHECK_LE(p_diff, 127UL);
963
964 vp9_info->p_diff[vp9_info->num_ref_pics] = static_cast<uint8_t>(p_diff);
965 ++vp9_info->num_ref_pics;
966
967 max_ref_temporal_layer_id =
968 std::max(max_ref_temporal_layer_id, ref_buf.temporal_layer_id);
969 } else {
970 RTC_DCHECK(inter_layer_predicted);
971 // RTP spec only allows to use previous spatial layer for inter-layer
972 // prediction.
973 RTC_DCHECK_EQ(ref_buf.spatial_layer_id + 1, layer_id.spatial_layer_id);
974 }
975 }
976
977 vp9_info->temporal_up_switch =
978 (max_ref_temporal_layer_id <
979 static_cast<size_t>(layer_id.temporal_layer_id));
980}
981
982void VP9EncoderImpl::UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
983 const size_t pic_num) {
984 vpx_svc_layer_id_t layer_id = {0};
985 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
986
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200987 const bool is_key_frame =
988 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
989
990 RefFrameBuffer frame_buf(pic_num, layer_id.spatial_layer_id,
991 layer_id.temporal_layer_id);
992
993 if (is_key_frame && layer_id.spatial_layer_id == 0) {
994 // Key frame updates all ref buffers.
995 for (size_t i = 0; i < kNumVp9Buffers; ++i) {
996 ref_buf_[i] = frame_buf;
997 }
Sergey Silkin76be2952018-08-21 12:30:33 +0200998 } else if (is_svc_) {
999 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
1000 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
1001
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001002 if (enc_layer_conf.update_last[layer_id.spatial_layer_id]) {
1003 ref_buf_[enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id]] =
1004 frame_buf;
1005 }
1006
1007 if (enc_layer_conf.update_alt_ref[layer_id.spatial_layer_id]) {
1008 ref_buf_[enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id]] =
1009 frame_buf;
1010 }
1011
1012 if (enc_layer_conf.update_golden[layer_id.spatial_layer_id]) {
1013 ref_buf_[enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id]] =
1014 frame_buf;
1015 }
Sergey Silkin76be2952018-08-21 12:30:33 +02001016 } else {
1017 RTC_DCHECK_EQ(num_spatial_layers_, 1);
1018 RTC_DCHECK_EQ(num_temporal_layers_, 1);
1019 // In non-svc mode encoder doesn't provide reference list. Assume each frame
1020 // is reference and stored in buffer 0.
1021 ref_buf_[0] = frame_buf;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001022 }
1023}
1024
asaperssona9455ab2015-07-31 06:10:09 -07001025int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) {
asapersson86956de2016-01-26 01:05:20 -08001026 RTC_DCHECK_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT);
asaperssona9455ab2015-07-31 06:10:09 -07001027
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001028 if (pkt->data.frame.sz == 0) {
1029 // Ignore dropped frame.
1030 return WEBRTC_VIDEO_CODEC_OK;
1031 }
1032
Sergey Silkin07f80cc2018-04-09 13:11:59 +02001033 vpx_svc_layer_id_t layer_id = {0};
1034 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
1035
1036 const bool first_frame_in_picture = encoded_image_._length == 0;
1037 // Ensure we don't buffer layers of previous picture (superframe).
1038 RTC_DCHECK(first_frame_in_picture || layer_id.spatial_layer_id > 0);
1039
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001040 const bool end_of_picture = false;
1041 DeliverBufferedFrame(end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001042
asaperssond9f641e2016-01-21 01:11:35 -08001043 if (pkt->data.frame.sz > encoded_image_._size) {
1044 delete[] encoded_image_._buffer;
1045 encoded_image_._size = pkt->data.frame.sz;
1046 encoded_image_._buffer = new uint8_t[encoded_image_._size];
1047 }
asapersson86956de2016-01-26 01:05:20 -08001048 memcpy(encoded_image_._buffer, pkt->data.frame.buf, pkt->data.frame.sz);
1049 encoded_image_._length = pkt->data.frame.sz;
asaperssond9f641e2016-01-21 01:11:35 -08001050
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001051 const bool is_key_frame =
1052 (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
1053 // Ensure encoder issued key frame on request.
1054 RTC_DCHECK(is_key_frame || !force_key_frame_);
1055
asaperssona9455ab2015-07-31 06:10:09 -07001056 // Check if encoded frame is a key frame.
asapersson86956de2016-01-26 01:05:20 -08001057 encoded_image_._frameType = kVideoFrameDelta;
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001058 if (is_key_frame) {
Peter Boström49e196a2015-10-23 15:58:18 +02001059 encoded_image_._frameType = kVideoFrameKey;
Sergey Silkinbd0954e2018-05-03 14:14:09 +02001060 force_key_frame_ = false;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001061 }
asapersson86956de2016-01-26 01:05:20 -08001062 RTC_DCHECK_LE(encoded_image_._length, encoded_image_._size);
1063
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001064 memset(&codec_specific_, 0, sizeof(codec_specific_));
Niels Möllerd3b8c632018-08-27 15:33:42 +02001065 absl::optional<int> spatial_index;
1066 PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt,
1067 input_image_->timestamp(), first_frame_in_picture);
1068 encoded_image_.SetSpatialIndex(spatial_index);
asaperssona9455ab2015-07-31 06:10:09 -07001069
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +02001070 if (is_flexible_mode_) {
1071 UpdateReferenceBuffers(*pkt, pics_since_key_);
1072 }
1073
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001074 TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length);
Niels Möller23775882018-08-16 10:24:12 +02001075 encoded_image_.SetTimestamp(input_image_->timestamp());
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001076 encoded_image_.capture_time_ms_ = input_image_->render_time_ms();
1077 encoded_image_.rotation_ = input_image_->rotation();
Niels Möllere3cf3d02018-06-13 11:52:16 +02001078 encoded_image_.content_type_ = (codec_.mode == VideoCodecMode::kScreensharing)
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001079 ? VideoContentType::SCREENSHARE
1080 : VideoContentType::UNSPECIFIED;
1081 encoded_image_._encodedHeight =
1082 pkt->data.frame.height[layer_id.spatial_layer_id];
1083 encoded_image_._encodedWidth =
1084 pkt->data.frame.width[layer_id.spatial_layer_id];
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +02001085 encoded_image_.timing_.flags = VideoSendTiming::kInvalid;
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001086 int qp = -1;
1087 vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp);
1088 encoded_image_.qp_ = qp;
ilnik04f4d122017-06-19 07:18:55 -07001089
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001090 return WEBRTC_VIDEO_CODEC_OK;
1091}
1092
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001093void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) {
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001094 if (encoded_image_._length > 0) {
Sergey Silkinbc0f0d32018-04-24 21:29:14 +02001095 codec_specific_.codecSpecific.VP9.end_of_picture = end_of_picture;
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001096
1097 // No data partitioning in VP9, so 1 partition only.
1098 int part_idx = 0;
1099 RTPFragmentationHeader frag_info;
1100 frag_info.VerifyAndAllocateFragmentationHeader(1);
1101 frag_info.fragmentationOffset[part_idx] = 0;
1102 frag_info.fragmentationLength[part_idx] = encoded_image_._length;
1103 frag_info.fragmentationPlType[part_idx] = 0;
1104 frag_info.fragmentationTimeDiff[part_idx] = 0;
1105
1106 encoded_complete_callback_->OnEncodedImage(encoded_image_, &codec_specific_,
1107 &frag_info);
1108 encoded_image_._length = 0;
Sergey Silkind902d582018-05-18 17:31:19 +02001109
Sergey Silkin96f2c972018-09-05 21:07:17 +02001110 if (codec_.mode == VideoCodecMode::kScreensharing) {
1111 const uint8_t spatial_idx = encoded_image_.SpatialIndex().value_or(0);
1112 const uint32_t frame_timestamp_ms =
Niels Möller23775882018-08-16 10:24:12 +02001113 1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency;
Sergey Silkin96f2c972018-09-05 21:07:17 +02001114 framerate_controller_[spatial_idx].AddFrame(frame_timestamp_ms);
Sergey Silkind902d582018-05-18 17:31:19 +02001115 }
Sergey Silkin2a1f1832018-04-04 11:45:41 +02001116 }
1117}
1118
pkasting@chromium.org16825b12015-01-12 21:51:21 +00001119int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001120 return WEBRTC_VIDEO_CODEC_OK;
1121}
1122
1123int VP9EncoderImpl::RegisterEncodeCompleteCallback(
1124 EncodedImageCallback* callback) {
1125 encoded_complete_callback_ = callback;
1126 return WEBRTC_VIDEO_CODEC_OK;
1127}
1128
Peter Boströmb7d9a972015-12-18 16:01:11 +01001129const char* VP9EncoderImpl::ImplementationName() const {
1130 return "libvpx";
1131}
1132
Magnus Jedvert46a27652017-11-13 14:10:02 +01001133std::unique_ptr<VP9Decoder> VP9Decoder::Create() {
Karl Wiberg918f50c2018-07-05 11:40:33 +02001134 return absl::make_unique<VP9DecoderImpl>();
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001135}
1136
1137VP9DecoderImpl::VP9DecoderImpl()
sprang3958ed82017-08-17 08:12:10 -07001138 : decode_complete_callback_(nullptr),
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001139 inited_(false),
sprang3958ed82017-08-17 08:12:10 -07001140 decoder_(nullptr),
philipel9d7d75b2018-04-04 13:03:01 +02001141 key_frame_required_(true) {}
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001142
1143VP9DecoderImpl::~VP9DecoderImpl() {
1144 inited_ = true; // in order to do the actual release
1145 Release();
Henrik Boström9695d852015-05-06 10:42:15 +02001146 int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse();
1147 if (num_buffers_in_use > 0) {
1148 // The frame buffers are reference counted and frames are exposed after
1149 // decoding. There may be valid usage cases where previous frames are still
1150 // referenced after ~VP9DecoderImpl that is not a leak.
Mirko Bonadei675513b2017-11-09 11:09:25 +01001151 RTC_LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still "
1152 << "referenced during ~VP9DecoderImpl.";
Henrik Boström9695d852015-05-06 10:42:15 +02001153 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001154}
1155
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001156int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001157 int ret_val = Release();
1158 if (ret_val < 0) {
1159 return ret_val;
1160 }
sprang3958ed82017-08-17 08:12:10 -07001161 if (decoder_ == nullptr) {
pbos@webrtc.orge728ee02014-12-17 13:43:55 +00001162 decoder_ = new vpx_codec_ctx_t;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001163 }
philipelcce46fc2015-12-21 03:04:49 -08001164 vpx_codec_dec_cfg_t cfg;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001165 // Setting number of threads to a constant value (1)
1166 cfg.threads = 1;
1167 cfg.h = cfg.w = 0; // set after decode
1168 vpx_codec_flags_t flags = 0;
1169 if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
1170 return WEBRTC_VIDEO_CODEC_MEMORY;
1171 }
Henrik Boström9695d852015-05-06 10:42:15 +02001172
1173 if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) {
1174 return WEBRTC_VIDEO_CODEC_MEMORY;
1175 }
1176
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001177 inited_ = true;
1178 // Always start with a complete key frame.
1179 key_frame_required_ = true;
1180 return WEBRTC_VIDEO_CODEC_OK;
1181}
1182
1183int VP9DecoderImpl::Decode(const EncodedImage& input_image,
1184 bool missing_frames,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001185 const CodecSpecificInfo* codec_specific_info,
1186 int64_t /*render_time_ms*/) {
1187 if (!inited_) {
1188 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1189 }
sprang3958ed82017-08-17 08:12:10 -07001190 if (decode_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001191 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1192 }
1193 // Always start with a complete key frame.
1194 if (key_frame_required_) {
Peter Boström49e196a2015-10-23 15:58:18 +02001195 if (input_image._frameType != kVideoFrameKey)
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001196 return WEBRTC_VIDEO_CODEC_ERROR;
1197 // We have a key frame - is it complete?
1198 if (input_image._completeFrame) {
1199 key_frame_required_ = false;
1200 } else {
1201 return WEBRTC_VIDEO_CODEC_ERROR;
1202 }
1203 }
sprang3958ed82017-08-17 08:12:10 -07001204 vpx_codec_iter_t iter = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001205 vpx_image_t* img;
1206 uint8_t* buffer = input_image._buffer;
1207 if (input_image._length == 0) {
sprang3958ed82017-08-17 08:12:10 -07001208 buffer = nullptr; // Triggers full frame concealment.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001209 }
Henrik Boström9695d852015-05-06 10:42:15 +02001210 // During decode libvpx may get and release buffers from |frame_buffer_pool_|.
1211 // In practice libvpx keeps a few (~3-4) buffers alive at a time.
philipelcce46fc2015-12-21 03:04:49 -08001212 if (vpx_codec_decode(decoder_, buffer,
1213 static_cast<unsigned int>(input_image._length), 0,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001214 VPX_DL_REALTIME)) {
1215 return WEBRTC_VIDEO_CODEC_ERROR;
1216 }
Henrik Boström9695d852015-05-06 10:42:15 +02001217 // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer.
1218 // It may be released by libvpx during future vpx_codec_decode or
1219 // vpx_codec_destroy calls.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001220 img = vpx_codec_get_frame(decoder_, &iter);
sakal7adadb12017-02-23 02:54:57 -08001221 int qp;
1222 vpx_codec_err_t vpx_ret =
1223 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
1224 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
1225 int ret =
Niels Möller23775882018-08-16 10:24:12 +02001226 ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_, qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001227 if (ret != 0) {
1228 return ret;
1229 }
1230 return WEBRTC_VIDEO_CODEC_OK;
1231}
1232
asapersson1490f7a2016-09-23 02:09:46 -07001233int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img,
1234 uint32_t timestamp,
sakal7adadb12017-02-23 02:54:57 -08001235 int64_t ntp_time_ms,
1236 int qp) {
sprang3958ed82017-08-17 08:12:10 -07001237 if (img == nullptr) {
1238 // Decoder OK and nullptr image => No show frame.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001239 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1240 }
Henrik Boström9695d852015-05-06 10:42:15 +02001241
1242 // This buffer contains all of |img|'s image data, a reference counted
perkj14f41442015-11-30 22:15:45 -08001243 // Vp9FrameBuffer. (libvpx is done with the buffers after a few
Henrik Boström9695d852015-05-06 10:42:15 +02001244 // vpx_codec_decode calls or vpx_codec_destroy).
1245 Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer =
1246 static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv);
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001247
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -07001248 // The buffer can be used directly by the VideoFrame (without copy) by
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001249 // using a Wrapped*Buffer.
1250 rtc::scoped_refptr<VideoFrameBuffer> img_wrapped_buffer;
1251 switch (img->bit_depth) {
1252 case 8:
1253 img_wrapped_buffer = WrapI420Buffer(
philipelcce46fc2015-12-21 03:04:49 -08001254 img->d_w, img->d_h, img->planes[VPX_PLANE_Y],
1255 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U],
1256 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V],
1257 img->stride[VPX_PLANE_V],
Henrik Boström9695d852015-05-06 10:42:15 +02001258 // WrappedI420Buffer's mechanism for allowing the release of its frame
1259 // buffer is through a callback function. This is where we should
1260 // release |img_buffer|.
Emircan Uysaler0823eec2018-07-13 17:10:00 -07001261 rtc::KeepRefUntilDone(img_buffer));
1262 break;
1263 case 10:
1264 img_wrapped_buffer = WrapI010Buffer(
1265 img->d_w, img->d_h,
1266 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]),
1267 img->stride[VPX_PLANE_Y] / 2,
1268 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]),
1269 img->stride[VPX_PLANE_U] / 2,
1270 reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]),
1271 img->stride[VPX_PLANE_V] / 2, rtc::KeepRefUntilDone(img_buffer));
1272 break;
1273 default:
1274 RTC_NOTREACHED();
1275 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1276 }
nisseca6d5d12016-06-17 05:03:04 -07001277
Emircan Uysaler715cc232018-07-23 17:50:43 -07001278 VideoFrame decoded_image = VideoFrame::Builder()
1279 .set_video_frame_buffer(img_wrapped_buffer)
1280 .set_timestamp_ms(0)
1281 .set_timestamp_rtp(timestamp)
1282 .set_ntp_time_ms(ntp_time_ms)
1283 .set_rotation(webrtc::kVideoRotation_0)
1284 .set_color_space(ExtractVP9ColorSpace(
1285 img->cs, img->range, img->bit_depth))
1286 .build();
Danil Chapovalov0040b662018-06-18 10:48:16 +02001287 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001288 return WEBRTC_VIDEO_CODEC_OK;
1289}
1290
1291int VP9DecoderImpl::RegisterDecodeCompleteCallback(
1292 DecodedImageCallback* callback) {
1293 decode_complete_callback_ = callback;
1294 return WEBRTC_VIDEO_CODEC_OK;
1295}
1296
1297int VP9DecoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001298 int ret_val = WEBRTC_VIDEO_CODEC_OK;
1299
sprang3958ed82017-08-17 08:12:10 -07001300 if (decoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +01001301 if (inited_) {
1302 // When a codec is destroyed libvpx will release any buffers of
1303 // |frame_buffer_pool_| it is currently using.
1304 if (vpx_codec_destroy(decoder_)) {
1305 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
1306 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001307 }
1308 delete decoder_;
sprang3958ed82017-08-17 08:12:10 -07001309 decoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001310 }
Henrik Boström9695d852015-05-06 10:42:15 +02001311 // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
1312 // still referenced externally are deleted once fully released, not returning
1313 // to the pool.
1314 frame_buffer_pool_.ClearPool();
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001315 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001316 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001317}
Peter Boströmb7d9a972015-12-18 16:01:11 +01001318
1319const char* VP9DecoderImpl::ImplementationName() const {
1320 return "libvpx";
1321}
1322
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001323} // namespace webrtc