blob: f2f88bbde01af32125ee392a95c2bdbe563a5817 [file] [log] [blame]
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020012#include "modules/video_coding/codecs/vp9/vp9_impl.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000013
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020014#include <algorithm>
Sergey Silkind45b3452018-05-31 16:00:24 +020015#include <limits>
marpan@webrtc.org5b883172014-11-01 06:10:48 +000016#include <vector>
17
marpan@webrtc.org5b883172014-11-01 06:10:48 +000018#include "vpx/vp8cx.h"
19#include "vpx/vp8dx.h"
Yves Gerey665174f2018-06-19 15:03:05 +020020#include "vpx/vpx_decoder.h"
21#include "vpx/vpx_encoder.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000022
Karl Wiberg918f50c2018-07-05 11:40:33 +020023#include "absl/memory/memory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "common_video/include/video_frame_buffer.h"
25#include "common_video/libyuv/include/webrtc_libyuv.h"
Sergey Silkind902d582018-05-18 17:31:19 +020026#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Sergey Silkin86684962018-03-28 19:32:37 +020027#include "modules/video_coding/codecs/vp9/svc_rate_allocator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "rtc_base/checks.h"
29#include "rtc_base/keep_ref_until_done.h"
30#include "rtc_base/logging.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "rtc_base/timeutils.h"
32#include "rtc_base/trace_event.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000033
34namespace webrtc {
35
Sergey Silkind902d582018-05-18 17:31:19 +020036namespace {
37const float kMaxScreenSharingFramerateFps = 5.0f;
38}
39
Marco6e89b252015-07-07 14:40:38 -070040// Only positive speeds, range for real-time coding currently is: 5 - 8.
41// Lower means slower/better quality, higher means fastest/lower quality.
42int GetCpuSpeed(int width, int height) {
Alex Glaznevfecb7c32016-03-31 14:23:27 -070043#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID)
Marco002f0d02015-12-17 09:49:31 -080044 return 8;
45#else
Marco6e89b252015-07-07 14:40:38 -070046 // For smaller resolutions, use lower speed setting (get some coding gain at
47 // the cost of increased encoding complexity).
48 if (width * height <= 352 * 288)
49 return 5;
50 else
51 return 7;
Marco002f0d02015-12-17 09:49:31 -080052#endif
Marco6e89b252015-07-07 14:40:38 -070053}
54
Emircan Uysaler98badbc2018-06-28 10:59:02 -070055std::vector<SdpVideoFormat> SupportedVP9Codecs() {
Emircan Uysalerc528c0a2018-07-13 21:13:20 +000056 return {SdpVideoFormat(
57 cricket::kVp9CodecName,
58 {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}}),
59 SdpVideoFormat(cricket::kVp9CodecName,
60 {{kVP9FmtpProfileId,
61 VP9ProfileToString(VP9Profile::kProfile2)}})};
Peter Boström12996152016-05-14 02:03:18 +020062}
63
Magnus Jedvert46a27652017-11-13 14:10:02 +010064std::unique_ptr<VP9Encoder> VP9Encoder::Create() {
Karl Wiberg918f50c2018-07-05 11:40:33 +020065 return absl::make_unique<VP9EncoderImpl>(cricket::VideoCodec());
Emircan Uysaler98badbc2018-06-28 10:59:02 -070066}
67
68std::unique_ptr<VP9Encoder> VP9Encoder::Create(
69 const cricket::VideoCodec& codec) {
Karl Wiberg918f50c2018-07-05 11:40:33 +020070 return absl::make_unique<VP9EncoderImpl>(codec);
marpan@webrtc.org5b883172014-11-01 06:10:48 +000071}
72
asaperssona9455ab2015-07-31 06:10:09 -070073void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
74 void* user_data) {
philipelcce46fc2015-12-21 03:04:49 -080075 VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data);
asaperssona9455ab2015-07-31 06:10:09 -070076 enc->GetEncodedLayerFrame(pkt);
77}
78
Emircan Uysaler98badbc2018-06-28 10:59:02 -070079VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec)
marpan@webrtc.org5b883172014-11-01 06:10:48 +000080 : encoded_image_(),
sprang3958ed82017-08-17 08:12:10 -070081 encoded_complete_callback_(nullptr),
Emircan Uysaler98badbc2018-06-28 10:59:02 -070082 profile_(
83 ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)),
marpan@webrtc.org5b883172014-11-01 06:10:48 +000084 inited_(false),
85 timestamp_(0),
marpan@webrtc.org5b883172014-11-01 06:10:48 +000086 cpu_speed_(3),
87 rc_max_intra_target_(0),
sprang3958ed82017-08-17 08:12:10 -070088 encoder_(nullptr),
89 config_(nullptr),
90 raw_(nullptr),
91 input_image_(nullptr),
Sergey Silkinbd0954e2018-05-03 14:14:09 +020092 force_key_frame_(true),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +020093 pics_since_key_(0),
asaperssona9455ab2015-07-31 06:10:09 -070094 num_temporal_layers_(0),
philipelcfc319b2015-11-10 07:17:23 -080095 num_spatial_layers_(0),
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020096 is_svc_(false),
Sergey Silkin6a8f30e2018-04-26 11:03:49 +020097 inter_layer_pred_(InterLayerPredMode::kOn),
Sergey Silkind902d582018-05-18 17:31:19 +020098 output_framerate_(1000.0, 1000.0),
99 last_encoded_frame_rtp_timestamp_(0),
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200100 is_flexible_mode_(false) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000101 memset(&codec_, 0, sizeof(codec_));
johannkoenig8225c402017-01-26 13:23:44 -0800102 memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t));
Emircan Uysalerc528c0a2018-07-13 21:13:20 +0000103 RTC_DCHECK(VP9Profile::kProfile0 == profile_);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000104}
105
106VP9EncoderImpl::~VP9EncoderImpl() {
107 Release();
108}
109
110int VP9EncoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100111 int ret_val = WEBRTC_VIDEO_CODEC_OK;
112
sprang3958ed82017-08-17 08:12:10 -0700113 if (encoded_image_._buffer != nullptr) {
philipelcce46fc2015-12-21 03:04:49 -0800114 delete[] encoded_image_._buffer;
sprang3958ed82017-08-17 08:12:10 -0700115 encoded_image_._buffer = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000116 }
sprang3958ed82017-08-17 08:12:10 -0700117 if (encoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +0100118 if (inited_) {
119 if (vpx_codec_destroy(encoder_)) {
120 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
121 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000122 }
123 delete encoder_;
sprang3958ed82017-08-17 08:12:10 -0700124 encoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000125 }
sprang3958ed82017-08-17 08:12:10 -0700126 if (config_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000127 delete config_;
sprang3958ed82017-08-17 08:12:10 -0700128 config_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000129 }
sprang3958ed82017-08-17 08:12:10 -0700130 if (raw_ != nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000131 vpx_img_free(raw_);
sprang3958ed82017-08-17 08:12:10 -0700132 raw_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000133 }
134 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +0100135 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000136}
137
sprangce4aef12015-11-02 07:23:20 -0800138bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const {
139 // We check target_bitrate_bps of the 0th layer to see if the spatial layers
140 // (i.e. bitrates) were explicitly configured.
Sergey Silkin13e74342018-03-02 12:28:00 +0100141 return num_spatial_layers_ > 1 && codec_.spatialLayers[0].targetBitrate > 0;
sprangce4aef12015-11-02 07:23:20 -0800142}
143
Erik Språng566124a2018-04-23 12:32:22 +0200144bool VP9EncoderImpl::SetSvcRates(
145 const VideoBitrateAllocation& bitrate_allocation) {
asaperssona9455ab2015-07-31 06:10:09 -0700146 uint8_t i = 0;
147
Sergey Silkin86684962018-03-28 19:32:37 +0200148 config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps();
Sergey Silkin86684962018-03-28 19:32:37 +0200149
“Michael23c5a992018-06-21 11:07:21 -0500150 num_active_spatial_layers_ = 0;
151 for (i = 0; i < num_spatial_layers_; ++i)
152 num_active_spatial_layers_ += bitrate_allocation.IsSpatialLayerUsed(i);
153 RTC_DCHECK_GT(num_active_spatial_layers_, 0);
154 RTC_DCHECK_LE(num_active_spatial_layers_, num_spatial_layers_);
155
sprangce4aef12015-11-02 07:23:20 -0800156 if (ExplicitlyConfiguredSpatialLayers()) {
Sergey Silkin86684962018-03-28 19:32:37 +0200157 for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) {
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200158 const bool was_layer_enabled = (config_->ss_target_bitrate[sl_idx] > 0);
Sergey Silkin86684962018-03-28 19:32:37 +0200159 config_->ss_target_bitrate[sl_idx] =
160 bitrate_allocation.GetSpatialLayerSum(sl_idx) / 1000;
161
162 for (size_t tl_idx = 0; tl_idx < num_temporal_layers_; ++tl_idx) {
163 config_->layer_target_bitrate[sl_idx * num_temporal_layers_ + tl_idx] =
164 bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000;
165 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200166
167 const bool is_layer_enabled = (config_->ss_target_bitrate[sl_idx] > 0);
168 if (is_layer_enabled && !was_layer_enabled) {
169 if (inter_layer_pred_ == InterLayerPredMode::kOff ||
170 inter_layer_pred_ == InterLayerPredMode::kOnKeyPic) {
171 // TODO(wemb:1526): remove key frame request when issue is fixed.
172 force_key_frame_ = true;
173 }
174 }
sprangce4aef12015-11-02 07:23:20 -0800175 }
176 } else {
177 float rate_ratio[VPX_MAX_LAYERS] = {0};
178 float total = 0;
“Michael67c8bcf2018-06-27 04:24:13 -0500179 for (i = 0; i < num_spatial_layers_; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800180 if (svc_params_.scaling_factor_num[i] <= 0 ||
181 svc_params_.scaling_factor_den[i] <= 0) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100182 RTC_LOG(LS_ERROR) << "Scaling factors not specified!";
sprangce4aef12015-11-02 07:23:20 -0800183 return false;
184 }
Yves Gerey665174f2018-06-19 15:03:05 +0200185 rate_ratio[i] = static_cast<float>(svc_params_.scaling_factor_num[i]) /
186 svc_params_.scaling_factor_den[i];
sprangce4aef12015-11-02 07:23:20 -0800187 total += rate_ratio[i];
188 }
189
“Michael67c8bcf2018-06-27 04:24:13 -0500190 for (i = 0; i < num_spatial_layers_; ++i) {
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200191 RTC_CHECK_GT(total, 0);
sprangce4aef12015-11-02 07:23:20 -0800192 config_->ss_target_bitrate[i] = static_cast<unsigned int>(
193 config_->rc_target_bitrate * rate_ratio[i] / total);
194 if (num_temporal_layers_ == 1) {
195 config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i];
196 } else if (num_temporal_layers_ == 2) {
197 config_->layer_target_bitrate[i * num_temporal_layers_] =
198 config_->ss_target_bitrate[i] * 2 / 3;
199 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
200 config_->ss_target_bitrate[i];
201 } else if (num_temporal_layers_ == 3) {
202 config_->layer_target_bitrate[i * num_temporal_layers_] =
203 config_->ss_target_bitrate[i] / 2;
204 config_->layer_target_bitrate[i * num_temporal_layers_ + 1] =
205 config_->layer_target_bitrate[i * num_temporal_layers_] +
206 (config_->ss_target_bitrate[i] / 4);
207 config_->layer_target_bitrate[i * num_temporal_layers_ + 2] =
208 config_->ss_target_bitrate[i];
209 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100210 RTC_LOG(LS_ERROR) << "Unsupported number of temporal layers: "
211 << num_temporal_layers_;
sprangce4aef12015-11-02 07:23:20 -0800212 return false;
213 }
asaperssona9455ab2015-07-31 06:10:09 -0700214 }
215 }
asaperssona9455ab2015-07-31 06:10:09 -0700216 return true;
217}
218
Erik Språng08127a92016-11-16 16:41:30 +0100219int VP9EncoderImpl::SetRateAllocation(
Erik Språng566124a2018-04-23 12:32:22 +0200220 const VideoBitrateAllocation& bitrate_allocation,
Erik Språng08127a92016-11-16 16:41:30 +0100221 uint32_t frame_rate) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000222 if (!inited_) {
223 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
224 }
225 if (encoder_->err) {
226 return WEBRTC_VIDEO_CODEC_ERROR;
227 }
Erik Språng08127a92016-11-16 16:41:30 +0100228 if (frame_rate < 1) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000229 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
230 }
231 // Update bit rate
Erik Språng08127a92016-11-16 16:41:30 +0100232 if (codec_.maxBitrate > 0 &&
233 bitrate_allocation.get_sum_kbps() > codec_.maxBitrate) {
234 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000235 }
Erik Språng08127a92016-11-16 16:41:30 +0100236
Erik Språng08127a92016-11-16 16:41:30 +0100237 codec_.maxFramerate = frame_rate;
asaperssona9455ab2015-07-31 06:10:09 -0700238
Sergey Silkin86684962018-03-28 19:32:37 +0200239 if (!SetSvcRates(bitrate_allocation)) {
asaperssona9455ab2015-07-31 06:10:09 -0700240 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
241 }
242
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000243 // Update encoder context
244 if (vpx_codec_enc_config_set(encoder_, config_)) {
245 return WEBRTC_VIDEO_CODEC_ERROR;
246 }
247 return WEBRTC_VIDEO_CODEC_OK;
248}
249
250int VP9EncoderImpl::InitEncode(const VideoCodec* inst,
251 int number_of_cores,
pkasting@chromium.org4591fbd2014-11-20 22:28:14 +0000252 size_t /*max_payload_size*/) {
sprang3958ed82017-08-17 08:12:10 -0700253 if (inst == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000254 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
255 }
256 if (inst->maxFramerate < 1) {
257 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
258 }
259 // Allow zero to represent an unspecified maxBitRate
260 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
261 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
262 }
263 if (inst->width < 1 || inst->height < 1) {
264 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
265 }
266 if (number_of_cores < 1) {
267 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
268 }
hta257dc392016-10-25 09:05:06 -0700269 if (inst->VP9().numberOfTemporalLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700270 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
271 }
ilnik2a8c2f52017-02-15 02:23:28 -0800272 // libvpx probably does not support more than 3 spatial layers.
273 if (inst->VP9().numberOfSpatialLayers > 3) {
asaperssona9455ab2015-07-31 06:10:09 -0700274 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
275 }
philipelcfc319b2015-11-10 07:17:23 -0800276
asapersson86956de2016-01-26 01:05:20 -0800277 int ret_val = Release();
278 if (ret_val < 0) {
279 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000280 }
sprang3958ed82017-08-17 08:12:10 -0700281 if (encoder_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000282 encoder_ = new vpx_codec_ctx_t;
283 }
sprang3958ed82017-08-17 08:12:10 -0700284 if (config_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000285 config_ = new vpx_codec_enc_cfg_t;
286 }
287 timestamp_ = 0;
288 if (&codec_ != inst) {
289 codec_ = *inst;
290 }
asaperssona9455ab2015-07-31 06:10:09 -0700291
hta257dc392016-10-25 09:05:06 -0700292 num_spatial_layers_ = inst->VP9().numberOfSpatialLayers;
Niels Möller65fb4042018-04-25 14:46:06 +0200293 RTC_DCHECK_GT(num_spatial_layers_, 0);
hta257dc392016-10-25 09:05:06 -0700294 num_temporal_layers_ = inst->VP9().numberOfTemporalLayers;
asaperssona9455ab2015-07-31 06:10:09 -0700295 if (num_temporal_layers_ == 0)
296 num_temporal_layers_ = 1;
297
Sergey Silkind902d582018-05-18 17:31:19 +0200298 // Init framerate controller.
299 output_framerate_.Reset();
Niels Möllere3cf3d02018-06-13 11:52:16 +0200300 if (codec_.mode == VideoCodecMode::kScreensharing) {
Sergey Silkind902d582018-05-18 17:31:19 +0200301 target_framerate_fps_ = kMaxScreenSharingFramerateFps;
302 } else {
303 target_framerate_fps_.reset();
304 }
305
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200306 is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1);
307 // Flexible mode requires SVC to be enabled since libvpx API only allows
308 // to get reference list in SVC mode.
309 RTC_DCHECK(!inst->VP9().flexibleMode || is_svc_);
310
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000311 // Allocate memory for encoded image
sprang3958ed82017-08-17 08:12:10 -0700312 if (encoded_image_._buffer != nullptr) {
philipelcce46fc2015-12-21 03:04:49 -0800313 delete[] encoded_image_._buffer;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000314 }
nisseeb44b392017-04-28 07:18:05 -0700315 encoded_image_._size =
316 CalcBufferSize(VideoType::kI420, codec_.width, codec_.height);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000317 encoded_image_._buffer = new uint8_t[encoded_image_._size];
318 encoded_image_._completeFrame = true;
Emircan Uysalerc528c0a2018-07-13 21:13:20 +0000319 // Creating a wrapper to the image - setting image data to nullptr. Actual
320 // pointer will be set in encode. Setting align to 1, as it is meaningless
321 // (actual memory is not allocated).
322 raw_ = vpx_img_wrap(nullptr, VPX_IMG_FMT_I420, codec_.width, codec_.height, 1,
323 nullptr);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000324 // Populate encoder configuration with default values.
325 if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) {
326 return WEBRTC_VIDEO_CODEC_ERROR;
327 }
328 config_->g_w = codec_.width;
329 config_->g_h = codec_.height;
330 config_->rc_target_bitrate = inst->startBitrate; // in kbit/s
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200331 config_->g_error_resilient = is_svc_ ? VPX_ERROR_RESILIENT_DEFAULT : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000332 // Setting the time base of the codec.
333 config_->g_timebase.num = 1;
334 config_->g_timebase.den = 90000;
335 config_->g_lag_in_frames = 0; // 0- no frame lagging
336 config_->g_threads = 1;
337 // Rate control settings.
hta257dc392016-10-25 09:05:06 -0700338 config_->rc_dropframe_thresh = inst->VP9().frameDroppingOn ? 30 : 0;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000339 config_->rc_end_usage = VPX_CBR;
340 config_->g_pass = VPX_RC_ONE_PASS;
341 config_->rc_min_quantizer = 2;
marpan@webrtc.orgdc8a9da2015-01-27 23:08:24 +0000342 config_->rc_max_quantizer = 52;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000343 config_->rc_undershoot_pct = 50;
344 config_->rc_overshoot_pct = 50;
345 config_->rc_buf_initial_sz = 500;
346 config_->rc_buf_optimal_sz = 600;
347 config_->rc_buf_sz = 1000;
348 // Set the maximum target size of any key-frame.
349 rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz);
hta257dc392016-10-25 09:05:06 -0700350 if (inst->VP9().keyFrameInterval > 0) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000351 config_->kf_mode = VPX_KF_AUTO;
hta257dc392016-10-25 09:05:06 -0700352 config_->kf_max_dist = inst->VP9().keyFrameInterval;
Åsa Perssonff24c042015-12-04 10:58:08 +0100353 // Needs to be set (in svc mode) to get correct periodic key frame interval
354 // (will have no effect in non-svc).
355 config_->kf_min_dist = config_->kf_max_dist;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000356 } else {
357 config_->kf_mode = VPX_KF_DISABLED;
358 }
hta257dc392016-10-25 09:05:06 -0700359 config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0;
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000360 // Determine number of threads based on the image size and #cores.
philipelcce46fc2015-12-21 03:04:49 -0800361 config_->g_threads =
362 NumberOfThreads(config_->g_w, config_->g_h, number_of_cores);
asaperssona9455ab2015-07-31 06:10:09 -0700363
Marco6e89b252015-07-07 14:40:38 -0700364 cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h);
asaperssona9455ab2015-07-31 06:10:09 -0700365
366 // TODO(asapersson): Check configuration of temporal switch up and increase
367 // pattern length.
hta257dc392016-10-25 09:05:06 -0700368 is_flexible_mode_ = inst->VP9().flexibleMode;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200369
370 // TODO(ssilkin): Only non-flexible mode is supported for now.
371 RTC_DCHECK(!is_flexible_mode_);
372
373 if (num_temporal_layers_ == 1) {
asaperssona9455ab2015-07-31 06:10:09 -0700374 gof_.SetGofInfoVP9(kTemporalStructureMode1);
375 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING;
376 config_->ts_number_layers = 1;
377 config_->ts_rate_decimator[0] = 1;
378 config_->ts_periodicity = 1;
379 config_->ts_layer_id[0] = 0;
380 } else if (num_temporal_layers_ == 2) {
381 gof_.SetGofInfoVP9(kTemporalStructureMode2);
382 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101;
383 config_->ts_number_layers = 2;
384 config_->ts_rate_decimator[0] = 2;
385 config_->ts_rate_decimator[1] = 1;
386 config_->ts_periodicity = 2;
387 config_->ts_layer_id[0] = 0;
388 config_->ts_layer_id[1] = 1;
389 } else if (num_temporal_layers_ == 3) {
390 gof_.SetGofInfoVP9(kTemporalStructureMode3);
391 config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212;
392 config_->ts_number_layers = 3;
393 config_->ts_rate_decimator[0] = 4;
394 config_->ts_rate_decimator[1] = 2;
395 config_->ts_rate_decimator[2] = 1;
396 config_->ts_periodicity = 4;
397 config_->ts_layer_id[0] = 0;
398 config_->ts_layer_id[1] = 2;
399 config_->ts_layer_id[2] = 1;
400 config_->ts_layer_id[3] = 2;
401 } else {
402 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
403 }
404
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200405 inter_layer_pred_ = inst->VP9().interLayerPred;
406
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200407 ref_buf_.clear();
408
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000409 return InitAndSetControlSettings(inst);
410}
411
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000412int VP9EncoderImpl::NumberOfThreads(int width,
413 int height,
414 int number_of_cores) {
415 // Keep the number of encoder threads equal to the possible number of column
416 // tiles, which is (1, 2, 4, 8). See comments below for VP9E_SET_TILE_COLUMNS.
417 if (width * height >= 1280 * 720 && number_of_cores > 4) {
418 return 4;
jianj23173a32017-07-12 16:11:09 -0700419 } else if (width * height >= 640 * 360 && number_of_cores > 2) {
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000420 return 2;
421 } else {
Yves Gerey665174f2018-06-19 15:03:05 +0200422// Use 2 threads for low res on ARM.
Jerome Jiang831af372017-12-05 10:44:35 -0800423#if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \
424 defined(WEBRTC_ANDROID)
425 if (width * height >= 320 * 180 && number_of_cores > 2) {
426 return 2;
427 }
428#endif
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000429 // 1 thread less than VGA.
430 return 1;
431 }
432}
433
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000434int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) {
Åsa Perssonff24c042015-12-04 10:58:08 +0100435 // Set QP-min/max per spatial and temporal layer.
436 int tot_num_layers = num_spatial_layers_ * num_temporal_layers_;
437 for (int i = 0; i < tot_num_layers; ++i) {
johannkoenig8225c402017-01-26 13:23:44 -0800438 svc_params_.max_quantizers[i] = config_->rc_max_quantizer;
439 svc_params_.min_quantizers[i] = config_->rc_min_quantizer;
Åsa Perssonff24c042015-12-04 10:58:08 +0100440 }
asaperssona9455ab2015-07-31 06:10:09 -0700441 config_->ss_number_layers = num_spatial_layers_;
sprangce4aef12015-11-02 07:23:20 -0800442 if (ExplicitlyConfiguredSpatialLayers()) {
443 for (int i = 0; i < num_spatial_layers_; ++i) {
444 const auto& layer = codec_.spatialLayers[i];
Rasmus Brandt58cd3852018-06-26 13:41:16 +0200445 RTC_CHECK_GT(layer.width, 0);
Sergey Silkin13e74342018-03-02 12:28:00 +0100446 const int scale_factor = codec_.width / layer.width;
447 RTC_DCHECK_GT(scale_factor, 0);
448
449 // Ensure scaler factor is integer.
450 if (scale_factor * layer.width != codec_.width) {
451 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
452 }
453
454 // Ensure scale factor is the same in both dimensions.
455 if (scale_factor * layer.height != codec_.height) {
456 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
457 }
458
459 // Ensure scale factor is power of two.
460 const bool is_pow_of_two = (scale_factor & (scale_factor - 1)) == 0;
461 if (!is_pow_of_two) {
462 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
463 }
464
465 svc_params_.scaling_factor_num[i] = 1;
466 svc_params_.scaling_factor_den[i] = scale_factor;
sprangce4aef12015-11-02 07:23:20 -0800467 }
468 } else {
469 int scaling_factor_num = 256;
470 for (int i = num_spatial_layers_ - 1; i >= 0; --i) {
sprangce4aef12015-11-02 07:23:20 -0800471 // 1:2 scaling in each dimension.
johannkoenig8225c402017-01-26 13:23:44 -0800472 svc_params_.scaling_factor_num[i] = scaling_factor_num;
473 svc_params_.scaling_factor_den[i] = 256;
sprangce4aef12015-11-02 07:23:20 -0800474 }
asaperssona9455ab2015-07-31 06:10:09 -0700475 }
476
Sergey Silkin86684962018-03-28 19:32:37 +0200477 SvcRateAllocator init_allocator(codec_);
Erik Språng566124a2018-04-23 12:32:22 +0200478 VideoBitrateAllocation allocation = init_allocator.GetAllocation(
Sergey Silkin86684962018-03-28 19:32:37 +0200479 inst->startBitrate * 1000, inst->maxFramerate);
480 if (!SetSvcRates(allocation)) {
asaperssona9455ab2015-07-31 06:10:09 -0700481 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
482 }
483
Emircan Uysalerc528c0a2018-07-13 21:13:20 +0000484 if (vpx_codec_enc_init(encoder_, vpx_codec_vp9_cx(), config_, 0)) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000485 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
486 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000487 vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_);
488 vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT,
489 rc_max_intra_target_);
490 vpx_codec_control(encoder_, VP9E_SET_AQ_MODE,
hta257dc392016-10-25 09:05:06 -0700491 inst->VP9().adaptiveQpMode ? 3 : 0);
asaperssona9455ab2015-07-31 06:10:09 -0700492
jianj822e5932017-07-12 16:09:58 -0700493 vpx_codec_control(encoder_, VP9E_SET_FRAME_PARALLEL_DECODING, 0);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200494
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200495 if (is_svc_) {
496 vpx_codec_control(encoder_, VP9E_SET_SVC, 1);
497 vpx_codec_control(encoder_, VP9E_SET_SVC_PARAMETERS, &svc_params_);
asaperssona9455ab2015-07-31 06:10:09 -0700498 }
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200499
500 if (num_spatial_layers_ > 1) {
501 switch (inter_layer_pred_) {
502 case InterLayerPredMode::kOn:
503 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 0);
504 break;
505 case InterLayerPredMode::kOff:
506 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 1);
507 break;
508 case InterLayerPredMode::kOnKeyPic:
509 vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 2);
510 break;
511 default:
512 RTC_NOTREACHED();
513 }
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200514
Sergey Silkinee203362018-05-30 11:34:08 +0200515 // Configure encoder to drop entire superframe whenever it needs to drop
516 // a layer. This mode is prefered over per-layer dropping which causes
517 // quality flickering and is not compatible with RTP non-flexible mode.
518 vpx_svc_frame_drop_t svc_drop_frame;
519 memset(&svc_drop_frame, 0, sizeof(svc_drop_frame));
520 svc_drop_frame.framedrop_mode = FULL_SUPERFRAME_DROP;
Sergey Silkind45b3452018-05-31 16:00:24 +0200521 svc_drop_frame.max_consec_drop = std::numeric_limits<int>::max();
Sergey Silkinee203362018-05-30 11:34:08 +0200522 for (size_t i = 0; i < num_spatial_layers_; ++i) {
523 svc_drop_frame.framedrop_thresh[i] = config_->rc_dropframe_thresh;
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200524 }
Sergey Silkinee203362018-05-30 11:34:08 +0200525 vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200526 }
527
asaperssona9455ab2015-07-31 06:10:09 -0700528 // Register callback for getting each spatial layer.
529 vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = {
philipelcce46fc2015-12-21 03:04:49 -0800530 VP9EncoderImpl::EncoderOutputCodedPacketCallback,
531 reinterpret_cast<void*>(this)};
532 vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK,
533 reinterpret_cast<void*>(&cbp));
asaperssona9455ab2015-07-31 06:10:09 -0700534
marpan@webrtc.org38d11b82015-01-26 15:21:36 +0000535 // Control function to set the number of column tiles in encoding a frame, in
536 // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns.
537 // The number tile columns will be capped by the encoder based on image size
538 // (minimum width of tile column is 256 pixels, maximum is 4096).
539 vpx_codec_control(encoder_, VP9E_SET_TILE_COLUMNS, (config_->g_threads >> 1));
jianjcb5d1152017-03-28 23:56:08 -0700540
541 // Turn on row-based multithreading.
542 vpx_codec_control(encoder_, VP9E_SET_ROW_MT, 1);
jianj6bf57e32017-06-05 13:43:49 -0700543
Alex Glaznevfecb7c32016-03-31 14:23:27 -0700544#if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \
Yves Gerey665174f2018-06-19 15:03:05 +0200545 !defined(ANDROID)
jianj6bf57e32017-06-05 13:43:49 -0700546 // Do not enable the denoiser on ARM since optimization is pending.
547 // Denoiser is on by default on other platforms.
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000548 vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY,
hta257dc392016-10-25 09:05:06 -0700549 inst->VP9().denoisingOn ? 1 : 0);
marpan@webrtc.org16a87b92015-03-05 22:19:00 +0000550#endif
jianj6bf57e32017-06-05 13:43:49 -0700551
Niels Möllere3cf3d02018-06-13 11:52:16 +0200552 if (codec_.mode == VideoCodecMode::kScreensharing) {
ivica242d6382015-09-04 06:13:23 -0700553 // Adjust internal parameters to screen content.
554 vpx_codec_control(encoder_, VP9E_SET_TUNE_CONTENT, 1);
ivica242d6382015-09-04 06:13:23 -0700555 }
Marco2520e722015-09-16 14:05:00 -0700556 // Enable encoder skip of static/low content blocks.
557 vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000558 inited_ = true;
559 return WEBRTC_VIDEO_CODEC_OK;
560}
561
562uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) {
563 // Set max to the optimal buffer level (normalized by target BR),
564 // and scaled by a scale_par.
565 // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps].
566 // This value is presented in percentage of perFrameBw:
567 // perFrameBw = targetBR[Kbps] * 1000 / framerate.
568 // The target in % is as follows:
569 float scale_par = 0.5;
570 uint32_t target_pct =
571 optimal_buffer_size * scale_par * codec_.maxFramerate / 10;
572 // Don't go below 3 times the per frame bandwidth.
573 const uint32_t min_intra_size = 300;
philipelcce46fc2015-12-21 03:04:49 -0800574 return (target_pct < min_intra_size) ? min_intra_size : target_pct;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000575}
576
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700577int VP9EncoderImpl::Encode(const VideoFrame& input_image,
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000578 const CodecSpecificInfo* codec_specific_info,
pbos22993e12015-10-19 02:39:06 -0700579 const std::vector<FrameType>* frame_types) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000580 if (!inited_) {
581 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
582 }
sprang3958ed82017-08-17 08:12:10 -0700583 if (encoded_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000584 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
585 }
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200586
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000587 // We only support one stream at the moment.
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200588 if (frame_types && !frame_types->empty()) {
589 if ((*frame_types)[0] == kVideoFrameKey) {
590 force_key_frame_ = true;
591 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000592 }
Sergey Silkind902d582018-05-18 17:31:19 +0200593
Niels Möllere3cf3d02018-06-13 11:52:16 +0200594 if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) {
Sergey Silkind902d582018-05-18 17:31:19 +0200595 if (DropFrame(input_image.timestamp())) {
596 return WEBRTC_VIDEO_CODEC_OK;
597 }
598 }
599
kwiberg352444f2016-11-28 15:58:53 -0800600 RTC_DCHECK_EQ(input_image.width(), raw_->d_w);
601 RTC_DCHECK_EQ(input_image.height(), raw_->d_h);
asaperssona9455ab2015-07-31 06:10:09 -0700602
603 // Set input image for use in the callback.
604 // This was necessary since you need some information from input_image.
605 // You can save only the necessary information (such as timestamp) instead of
606 // doing this.
607 input_image_ = &input_image;
608
Emircan Uysalerc528c0a2018-07-13 21:13:20 +0000609 rtc::scoped_refptr<I420BufferInterface> i420_buffer =
610 input_image.video_frame_buffer()->ToI420();
611 // Image in vpx_image_t format.
612 // Input image is const. VPX's raw image is not defined as const.
613 raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY());
614 raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU());
615 raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV());
616 raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY();
617 raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU();
618 raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV();
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000619
philipelcfc319b2015-11-10 07:17:23 -0800620 vpx_enc_frame_flags_t flags = 0;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200621 if (force_key_frame_) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000622 flags = VPX_EFLAG_FORCE_KF;
623 }
philipelcfc319b2015-11-10 07:17:23 -0800624
sprang3958ed82017-08-17 08:12:10 -0700625 RTC_CHECK_GT(codec_.maxFramerate, 0);
Sergey Silkind902d582018-05-18 17:31:19 +0200626 uint32_t duration =
627 90000 / target_framerate_fps_.value_or(codec_.maxFramerate);
Emircan Uysalerc528c0a2018-07-13 21:13:20 +0000628 if (vpx_codec_encode(encoder_, raw_, timestamp_, duration, flags,
629 VPX_DL_REALTIME)) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000630 return WEBRTC_VIDEO_CODEC_ERROR;
631 }
632 timestamp_ += duration;
asaperssona9455ab2015-07-31 06:10:09 -0700633
Sergey Silkinbc0f0d32018-04-24 21:29:14 +0200634 const bool end_of_picture = true;
635 DeliverBufferedFrame(end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200636
asaperssona9455ab2015-07-31 06:10:09 -0700637 return WEBRTC_VIDEO_CODEC_OK;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000638}
639
640void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
philipelcce46fc2015-12-21 03:04:49 -0800641 const vpx_codec_cx_pkt& pkt,
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200642 uint32_t timestamp,
643 bool first_frame_in_picture) {
sprang3958ed82017-08-17 08:12:10 -0700644 RTC_CHECK(codec_specific != nullptr);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000645 codec_specific->codecType = kVideoCodecVP9;
perkj275afc52016-09-01 00:21:16 -0700646 codec_specific->codec_name = ImplementationName();
philipelcce46fc2015-12-21 03:04:49 -0800647 CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9);
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200648
649 vp9_info->first_frame_in_picture = first_frame_in_picture;
hta257dc392016-10-25 09:05:06 -0700650 vp9_info->flexible_mode = codec_.VP9()->flexibleMode;
651 vp9_info->ss_data_available =
652 ((pkt.data.frame.flags & VPX_FRAME_IS_KEY) && !codec_.VP9()->flexibleMode)
653 ? true
654 : false;
asaperssona9455ab2015-07-31 06:10:09 -0700655
656 vpx_svc_layer_id_t layer_id = {0};
657 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
658
sprang3958ed82017-08-17 08:12:10 -0700659 RTC_CHECK_GT(num_temporal_layers_, 0);
“Michael23c5a992018-06-21 11:07:21 -0500660 RTC_CHECK_GT(num_active_spatial_layers_, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700661 if (num_temporal_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700662 RTC_CHECK_EQ(layer_id.temporal_layer_id, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700663 vp9_info->temporal_idx = kNoTemporalIdx;
664 } else {
665 vp9_info->temporal_idx = layer_id.temporal_layer_id;
666 }
“Michael23c5a992018-06-21 11:07:21 -0500667 if (num_active_spatial_layers_ == 1) {
sprang3958ed82017-08-17 08:12:10 -0700668 RTC_CHECK_EQ(layer_id.spatial_layer_id, 0);
asaperssona9455ab2015-07-31 06:10:09 -0700669 vp9_info->spatial_idx = kNoSpatialIdx;
670 } else {
671 vp9_info->spatial_idx = layer_id.spatial_layer_id;
672 }
673 if (layer_id.spatial_layer_id != 0) {
674 vp9_info->ss_data_available = false;
675 }
676
asaperssona9455ab2015-07-31 06:10:09 -0700677 // TODO(asapersson): this info has to be obtained from the encoder.
asaperssoncb50c962015-11-18 01:58:55 -0800678 vp9_info->temporal_up_switch = false;
asaperssona9455ab2015-07-31 06:10:09 -0700679
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200680 if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) {
681 pics_since_key_ = 0;
682 } else if (first_frame_in_picture) {
683 ++pics_since_key_;
asaperssona9455ab2015-07-31 06:10:09 -0700684 }
685
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200686 const bool is_key_pic = (pics_since_key_ == 0);
687 const bool is_inter_layer_pred_allowed =
688 (inter_layer_pred_ == InterLayerPredMode::kOn ||
689 (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic));
690
691 // Always set inter_layer_predicted to true on high layer frame if inter-layer
692 // prediction (ILP) is allowed even if encoder didn't actually use it.
693 // Setting inter_layer_predicted to false would allow receiver to decode high
694 // layer frame without decoding low layer frame. If that would happen (e.g.
695 // if low layer frame is lost) then receiver won't be able to decode next high
696 // layer frame which uses ILP.
697 vp9_info->inter_layer_predicted =
698 first_frame_in_picture ? false : is_inter_layer_pred_allowed;
699
700 const bool is_last_layer =
“Michael23c5a992018-06-21 11:07:21 -0500701 (layer_id.spatial_layer_id + 1 == num_active_spatial_layers_);
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200702 vp9_info->non_ref_for_inter_layer_pred =
703 is_last_layer ? true : !is_inter_layer_pred_allowed;
asapersson00ac85e2015-11-11 05:30:48 -0800704
ivica7f6a6fc2015-09-08 02:40:29 -0700705 // Always populate this, so that the packetizer can properly set the marker
706 // bit.
“Michael23c5a992018-06-21 11:07:21 -0500707 vp9_info->num_spatial_layers = num_active_spatial_layers_;
philipelcfc319b2015-11-10 07:17:23 -0800708
Sergey Silkinbe71a1e2018-05-17 16:46:43 +0200709 RTC_DCHECK(!vp9_info->flexible_mode);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200710
711 vp9_info->num_ref_pics = 0;
712 if (vp9_info->flexible_mode) {
713 vp9_info->gof_idx = kNoGofIdx;
714 FillReferenceIndices(pkt, pics_since_key_, vp9_info->inter_layer_predicted,
715 vp9_info);
716 } else {
717 vp9_info->gof_idx =
718 static_cast<uint8_t>(pics_since_key_ % gof_.num_frames_in_gof);
719 vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx];
720 vp9_info->num_ref_pics = gof_.num_ref_pics[vp9_info->gof_idx];
721 }
722
723 vp9_info->inter_pic_predicted = (!is_key_pic && vp9_info->num_ref_pics > 0);
724
asaperssona9455ab2015-07-31 06:10:09 -0700725 if (vp9_info->ss_data_available) {
asaperssona9455ab2015-07-31 06:10:09 -0700726 vp9_info->spatial_layer_resolution_present = true;
“Michael23c5a992018-06-21 11:07:21 -0500727 for (size_t i = 0; i < num_active_spatial_layers_; ++i) {
Yves Gerey665174f2018-06-19 15:03:05 +0200728 vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -0800729 svc_params_.scaling_factor_den[i];
Yves Gerey665174f2018-06-19 15:03:05 +0200730 vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] /
johannkoenig8225c402017-01-26 13:23:44 -0800731 svc_params_.scaling_factor_den[i];
asaperssona9455ab2015-07-31 06:10:09 -0700732 }
733 if (!vp9_info->flexible_mode) {
734 vp9_info->gof.CopyGofInfoVP9(gof_);
735 }
736 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000737}
738
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200739void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
740 const size_t pic_num,
741 const bool inter_layer_predicted,
742 CodecSpecificInfoVP9* vp9_info) {
743 vpx_svc_layer_id_t layer_id = {0};
744 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
745
746 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
747 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
748
749 std::vector<RefFrameBuffer> ref_buf_list;
750 if (enc_layer_conf.reference_last[layer_id.spatial_layer_id]) {
751 const size_t fb_idx = enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id];
752 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
753 ref_buf_list.push_back(ref_buf_.at(fb_idx));
754 }
755
756 if (enc_layer_conf.reference_alt_ref[layer_id.spatial_layer_id]) {
757 const size_t fb_idx = enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id];
758 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
759 ref_buf_list.push_back(ref_buf_.at(fb_idx));
760 }
761
762 if (enc_layer_conf.reference_golden[layer_id.spatial_layer_id]) {
763 const size_t fb_idx = enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id];
764 RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end());
765 ref_buf_list.push_back(ref_buf_.at(fb_idx));
766 }
767
768 size_t max_ref_temporal_layer_id = 0;
769
770 vp9_info->num_ref_pics = 0;
771 for (const RefFrameBuffer& ref_buf : ref_buf_list) {
772 RTC_DCHECK_LE(ref_buf.pic_num, pic_num);
773 if (ref_buf.pic_num < pic_num) {
774 if (inter_layer_pred_ != InterLayerPredMode::kOn) {
775 // RTP spec limits temporal prediction to the same spatial layer.
776 // It is safe to ignore this requirement if inter-layer prediction is
777 // enabled for all frames when all base frames are relayed to receiver.
778 RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id);
779 }
780 RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id);
781
782 const size_t p_diff = pic_num - ref_buf.pic_num;
783 RTC_DCHECK_LE(p_diff, 127UL);
784
785 vp9_info->p_diff[vp9_info->num_ref_pics] = static_cast<uint8_t>(p_diff);
786 ++vp9_info->num_ref_pics;
787
788 max_ref_temporal_layer_id =
789 std::max(max_ref_temporal_layer_id, ref_buf.temporal_layer_id);
790 } else {
791 RTC_DCHECK(inter_layer_predicted);
792 // RTP spec only allows to use previous spatial layer for inter-layer
793 // prediction.
794 RTC_DCHECK_EQ(ref_buf.spatial_layer_id + 1, layer_id.spatial_layer_id);
795 }
796 }
797
798 vp9_info->temporal_up_switch =
799 (max_ref_temporal_layer_id <
800 static_cast<size_t>(layer_id.temporal_layer_id));
801}
802
803void VP9EncoderImpl::UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
804 const size_t pic_num) {
805 vpx_svc_layer_id_t layer_id = {0};
806 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
807
808 vpx_svc_ref_frame_config_t enc_layer_conf = {{0}};
809 vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf);
810
811 const bool is_key_frame =
812 (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
813
814 RefFrameBuffer frame_buf(pic_num, layer_id.spatial_layer_id,
815 layer_id.temporal_layer_id);
816
817 if (is_key_frame && layer_id.spatial_layer_id == 0) {
818 // Key frame updates all ref buffers.
819 for (size_t i = 0; i < kNumVp9Buffers; ++i) {
820 ref_buf_[i] = frame_buf;
821 }
822 } else {
823 if (enc_layer_conf.update_last[layer_id.spatial_layer_id]) {
824 ref_buf_[enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id]] =
825 frame_buf;
826 }
827
828 if (enc_layer_conf.update_alt_ref[layer_id.spatial_layer_id]) {
829 ref_buf_[enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id]] =
830 frame_buf;
831 }
832
833 if (enc_layer_conf.update_golden[layer_id.spatial_layer_id]) {
834 ref_buf_[enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id]] =
835 frame_buf;
836 }
837 }
838}
839
asaperssona9455ab2015-07-31 06:10:09 -0700840int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) {
asapersson86956de2016-01-26 01:05:20 -0800841 RTC_DCHECK_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT);
asaperssona9455ab2015-07-31 06:10:09 -0700842
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200843 if (pkt->data.frame.sz == 0) {
844 // Ignore dropped frame.
845 return WEBRTC_VIDEO_CODEC_OK;
846 }
847
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200848 vpx_svc_layer_id_t layer_id = {0};
849 vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id);
850
851 const bool first_frame_in_picture = encoded_image_._length == 0;
852 // Ensure we don't buffer layers of previous picture (superframe).
853 RTC_DCHECK(first_frame_in_picture || layer_id.spatial_layer_id > 0);
854
Sergey Silkinbc0f0d32018-04-24 21:29:14 +0200855 const bool end_of_picture = false;
856 DeliverBufferedFrame(end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200857
asaperssond9f641e2016-01-21 01:11:35 -0800858 if (pkt->data.frame.sz > encoded_image_._size) {
859 delete[] encoded_image_._buffer;
860 encoded_image_._size = pkt->data.frame.sz;
861 encoded_image_._buffer = new uint8_t[encoded_image_._size];
862 }
asapersson86956de2016-01-26 01:05:20 -0800863 memcpy(encoded_image_._buffer, pkt->data.frame.buf, pkt->data.frame.sz);
864 encoded_image_._length = pkt->data.frame.sz;
asaperssond9f641e2016-01-21 01:11:35 -0800865
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200866 const bool is_key_frame =
867 (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false;
868 // Ensure encoder issued key frame on request.
869 RTC_DCHECK(is_key_frame || !force_key_frame_);
870
asaperssona9455ab2015-07-31 06:10:09 -0700871 // Check if encoded frame is a key frame.
asapersson86956de2016-01-26 01:05:20 -0800872 encoded_image_._frameType = kVideoFrameDelta;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200873 if (is_key_frame) {
Peter Boström49e196a2015-10-23 15:58:18 +0200874 encoded_image_._frameType = kVideoFrameKey;
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200875 force_key_frame_ = false;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000876 }
asapersson86956de2016-01-26 01:05:20 -0800877 RTC_DCHECK_LE(encoded_image_._length, encoded_image_._size);
878
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200879 memset(&codec_specific_, 0, sizeof(codec_specific_));
Sergey Silkin07f80cc2018-04-09 13:11:59 +0200880 PopulateCodecSpecific(&codec_specific_, *pkt, input_image_->timestamp(),
881 first_frame_in_picture);
asaperssona9455ab2015-07-31 06:10:09 -0700882
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200883 if (is_flexible_mode_) {
884 UpdateReferenceBuffers(*pkt, pics_since_key_);
885 }
886
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200887 TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length);
888 encoded_image_._timeStamp = input_image_->timestamp();
889 encoded_image_.capture_time_ms_ = input_image_->render_time_ms();
890 encoded_image_.rotation_ = input_image_->rotation();
Niels Möllere3cf3d02018-06-13 11:52:16 +0200891 encoded_image_.content_type_ = (codec_.mode == VideoCodecMode::kScreensharing)
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200892 ? VideoContentType::SCREENSHARE
893 : VideoContentType::UNSPECIFIED;
894 encoded_image_._encodedHeight =
895 pkt->data.frame.height[layer_id.spatial_layer_id];
896 encoded_image_._encodedWidth =
897 pkt->data.frame.width[layer_id.spatial_layer_id];
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +0200898 encoded_image_.timing_.flags = VideoSendTiming::kInvalid;
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200899 int qp = -1;
900 vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp);
901 encoded_image_.qp_ = qp;
ilnik04f4d122017-06-19 07:18:55 -0700902
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000903 return WEBRTC_VIDEO_CODEC_OK;
904}
905
Sergey Silkinbc0f0d32018-04-24 21:29:14 +0200906void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) {
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200907 if (encoded_image_._length > 0) {
Sergey Silkinbc0f0d32018-04-24 21:29:14 +0200908 codec_specific_.codecSpecific.VP9.end_of_picture = end_of_picture;
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200909
910 // No data partitioning in VP9, so 1 partition only.
911 int part_idx = 0;
912 RTPFragmentationHeader frag_info;
913 frag_info.VerifyAndAllocateFragmentationHeader(1);
914 frag_info.fragmentationOffset[part_idx] = 0;
915 frag_info.fragmentationLength[part_idx] = encoded_image_._length;
916 frag_info.fragmentationPlType[part_idx] = 0;
917 frag_info.fragmentationTimeDiff[part_idx] = 0;
918
919 encoded_complete_callback_->OnEncodedImage(encoded_image_, &codec_specific_,
920 &frag_info);
921 encoded_image_._length = 0;
Sergey Silkind902d582018-05-18 17:31:19 +0200922
923 if (end_of_picture) {
924 const uint32_t timestamp_ms =
925 1000 * encoded_image_._timeStamp / kVideoPayloadTypeFrequency;
926 output_framerate_.Update(1, timestamp_ms);
927 last_encoded_frame_rtp_timestamp_ = encoded_image_._timeStamp;
928 }
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200929 }
930}
931
Sergey Silkind902d582018-05-18 17:31:19 +0200932bool VP9EncoderImpl::DropFrame(uint32_t rtp_timestamp) {
933 if (target_framerate_fps_) {
934 if (rtp_timestamp < last_encoded_frame_rtp_timestamp_) {
935 // Timestamp has wrapped around. Reset framerate statistic.
936 output_framerate_.Reset();
937 return false;
938 }
939
940 const uint32_t timestamp_ms =
941 1000 * rtp_timestamp / kVideoPayloadTypeFrequency;
942 const uint32_t framerate_fps =
943 output_framerate_.Rate(timestamp_ms).value_or(0);
944 if (framerate_fps > *target_framerate_fps_) {
945 return true;
946 }
947
948 // Primarily check if frame interval is too short using frame timestamps,
949 // as if they are correct they won't be affected by queuing in webrtc.
950 const uint32_t expected_frame_interval =
951 kVideoPayloadTypeFrequency / *target_framerate_fps_;
952
953 const uint32_t ts_diff = rtp_timestamp - last_encoded_frame_rtp_timestamp_;
954 if (ts_diff < 85 * expected_frame_interval / 100) {
955 return true;
956 }
957 }
958
959 return false;
960}
961
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000962int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000963 return WEBRTC_VIDEO_CODEC_OK;
964}
965
966int VP9EncoderImpl::RegisterEncodeCompleteCallback(
967 EncodedImageCallback* callback) {
968 encoded_complete_callback_ = callback;
969 return WEBRTC_VIDEO_CODEC_OK;
970}
971
Peter Boströmb7d9a972015-12-18 16:01:11 +0100972const char* VP9EncoderImpl::ImplementationName() const {
973 return "libvpx";
974}
975
Magnus Jedvert46a27652017-11-13 14:10:02 +0100976std::unique_ptr<VP9Decoder> VP9Decoder::Create() {
Karl Wiberg918f50c2018-07-05 11:40:33 +0200977 return absl::make_unique<VP9DecoderImpl>();
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000978}
979
980VP9DecoderImpl::VP9DecoderImpl()
sprang3958ed82017-08-17 08:12:10 -0700981 : decode_complete_callback_(nullptr),
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000982 inited_(false),
sprang3958ed82017-08-17 08:12:10 -0700983 decoder_(nullptr),
philipel9d7d75b2018-04-04 13:03:01 +0200984 key_frame_required_(true) {}
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000985
986VP9DecoderImpl::~VP9DecoderImpl() {
987 inited_ = true; // in order to do the actual release
988 Release();
Henrik Boström9695d852015-05-06 10:42:15 +0200989 int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse();
990 if (num_buffers_in_use > 0) {
991 // The frame buffers are reference counted and frames are exposed after
992 // decoding. There may be valid usage cases where previous frames are still
993 // referenced after ~VP9DecoderImpl that is not a leak.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100994 RTC_LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still "
995 << "referenced during ~VP9DecoderImpl.";
Henrik Boström9695d852015-05-06 10:42:15 +0200996 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000997}
998
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000999int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001000 int ret_val = Release();
1001 if (ret_val < 0) {
1002 return ret_val;
1003 }
sprang3958ed82017-08-17 08:12:10 -07001004 if (decoder_ == nullptr) {
pbos@webrtc.orge728ee02014-12-17 13:43:55 +00001005 decoder_ = new vpx_codec_ctx_t;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001006 }
philipelcce46fc2015-12-21 03:04:49 -08001007 vpx_codec_dec_cfg_t cfg;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001008 // Setting number of threads to a constant value (1)
1009 cfg.threads = 1;
1010 cfg.h = cfg.w = 0; // set after decode
1011 vpx_codec_flags_t flags = 0;
1012 if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) {
1013 return WEBRTC_VIDEO_CODEC_MEMORY;
1014 }
Henrik Boström9695d852015-05-06 10:42:15 +02001015
1016 if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) {
1017 return WEBRTC_VIDEO_CODEC_MEMORY;
1018 }
1019
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001020 inited_ = true;
1021 // Always start with a complete key frame.
1022 key_frame_required_ = true;
1023 return WEBRTC_VIDEO_CODEC_OK;
1024}
1025
1026int VP9DecoderImpl::Decode(const EncodedImage& input_image,
1027 bool missing_frames,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001028 const CodecSpecificInfo* codec_specific_info,
1029 int64_t /*render_time_ms*/) {
1030 if (!inited_) {
1031 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1032 }
sprang3958ed82017-08-17 08:12:10 -07001033 if (decode_complete_callback_ == nullptr) {
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001034 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
1035 }
1036 // Always start with a complete key frame.
1037 if (key_frame_required_) {
Peter Boström49e196a2015-10-23 15:58:18 +02001038 if (input_image._frameType != kVideoFrameKey)
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001039 return WEBRTC_VIDEO_CODEC_ERROR;
1040 // We have a key frame - is it complete?
1041 if (input_image._completeFrame) {
1042 key_frame_required_ = false;
1043 } else {
1044 return WEBRTC_VIDEO_CODEC_ERROR;
1045 }
1046 }
sprang3958ed82017-08-17 08:12:10 -07001047 vpx_codec_iter_t iter = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001048 vpx_image_t* img;
1049 uint8_t* buffer = input_image._buffer;
1050 if (input_image._length == 0) {
sprang3958ed82017-08-17 08:12:10 -07001051 buffer = nullptr; // Triggers full frame concealment.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001052 }
Henrik Boström9695d852015-05-06 10:42:15 +02001053 // During decode libvpx may get and release buffers from |frame_buffer_pool_|.
1054 // In practice libvpx keeps a few (~3-4) buffers alive at a time.
philipelcce46fc2015-12-21 03:04:49 -08001055 if (vpx_codec_decode(decoder_, buffer,
1056 static_cast<unsigned int>(input_image._length), 0,
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001057 VPX_DL_REALTIME)) {
1058 return WEBRTC_VIDEO_CODEC_ERROR;
1059 }
Henrik Boström9695d852015-05-06 10:42:15 +02001060 // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer.
1061 // It may be released by libvpx during future vpx_codec_decode or
1062 // vpx_codec_destroy calls.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001063 img = vpx_codec_get_frame(decoder_, &iter);
sakal7adadb12017-02-23 02:54:57 -08001064 int qp;
1065 vpx_codec_err_t vpx_ret =
1066 vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp);
1067 RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK);
1068 int ret =
1069 ReturnFrame(img, input_image._timeStamp, input_image.ntp_time_ms_, qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001070 if (ret != 0) {
1071 return ret;
1072 }
1073 return WEBRTC_VIDEO_CODEC_OK;
1074}
1075
asapersson1490f7a2016-09-23 02:09:46 -07001076int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img,
1077 uint32_t timestamp,
sakal7adadb12017-02-23 02:54:57 -08001078 int64_t ntp_time_ms,
1079 int qp) {
sprang3958ed82017-08-17 08:12:10 -07001080 if (img == nullptr) {
1081 // Decoder OK and nullptr image => No show frame.
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001082 return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
1083 }
Henrik Boström9695d852015-05-06 10:42:15 +02001084
1085 // This buffer contains all of |img|'s image data, a reference counted
perkj14f41442015-11-30 22:15:45 -08001086 // Vp9FrameBuffer. (libvpx is done with the buffers after a few
Henrik Boström9695d852015-05-06 10:42:15 +02001087 // vpx_codec_decode calls or vpx_codec_destroy).
1088 Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer =
1089 static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv);
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -07001090 // The buffer can be used directly by the VideoFrame (without copy) by
Emircan Uysalerc528c0a2018-07-13 21:13:20 +00001091 // using a WrappedI420Buffer.
1092 rtc::scoped_refptr<WrappedI420Buffer> img_wrapped_buffer(
1093 new rtc::RefCountedObject<webrtc::WrappedI420Buffer>(
philipelcce46fc2015-12-21 03:04:49 -08001094 img->d_w, img->d_h, img->planes[VPX_PLANE_Y],
1095 img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U],
1096 img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V],
1097 img->stride[VPX_PLANE_V],
Henrik Boström9695d852015-05-06 10:42:15 +02001098 // WrappedI420Buffer's mechanism for allowing the release of its frame
1099 // buffer is through a callback function. This is where we should
1100 // release |img_buffer|.
Emircan Uysalerc528c0a2018-07-13 21:13:20 +00001101 rtc::KeepRefUntilDone(img_buffer)));
1102
nisseca6d5d12016-06-17 05:03:04 -07001103 VideoFrame decoded_image(img_wrapped_buffer, timestamp,
1104 0 /* render_time_ms */, webrtc::kVideoRotation_0);
asapersson1490f7a2016-09-23 02:09:46 -07001105 decoded_image.set_ntp_time_ms(ntp_time_ms);
nisseca6d5d12016-06-17 05:03:04 -07001106
Danil Chapovalov0040b662018-06-18 10:48:16 +02001107 decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001108 return WEBRTC_VIDEO_CODEC_OK;
1109}
1110
1111int VP9DecoderImpl::RegisterDecodeCompleteCallback(
1112 DecodedImageCallback* callback) {
1113 decode_complete_callback_ = callback;
1114 return WEBRTC_VIDEO_CODEC_OK;
1115}
1116
1117int VP9DecoderImpl::Release() {
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001118 int ret_val = WEBRTC_VIDEO_CODEC_OK;
1119
sprang3958ed82017-08-17 08:12:10 -07001120 if (decoder_ != nullptr) {
Sergey Silkin90399692018-03-02 14:44:10 +01001121 if (inited_) {
1122 // When a codec is destroyed libvpx will release any buffers of
1123 // |frame_buffer_pool_| it is currently using.
1124 if (vpx_codec_destroy(decoder_)) {
1125 ret_val = WEBRTC_VIDEO_CODEC_MEMORY;
1126 }
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001127 }
1128 delete decoder_;
sprang3958ed82017-08-17 08:12:10 -07001129 decoder_ = nullptr;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001130 }
Henrik Boström9695d852015-05-06 10:42:15 +02001131 // Releases buffers from the pool. Any buffers not in use are deleted. Buffers
1132 // still referenced externally are deleted once fully released, not returning
1133 // to the pool.
1134 frame_buffer_pool_.ClearPool();
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001135 inited_ = false;
Sergey Silkin3e871ea2018-03-02 13:11:04 +01001136 return ret_val;
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001137}
Peter Boströmb7d9a972015-12-18 16:01:11 +01001138
1139const char* VP9DecoderImpl::ImplementationName() const {
1140 return "libvpx";
1141}
1142
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001143} // namespace webrtc