marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | * |
| 10 | */ |
| 11 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 12 | #include "modules/video_coding/codecs/vp9/vp9_impl.h" |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 13 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 14 | #include <algorithm> |
Sergey Silkin | d45b345 | 2018-05-31 16:00:24 +0200 | [diff] [blame] | 15 | #include <limits> |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 16 | #include <vector> |
| 17 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 18 | #include "vpx/vp8cx.h" |
| 19 | #include "vpx/vp8dx.h" |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 20 | #include "vpx/vpx_decoder.h" |
| 21 | #include "vpx/vpx_encoder.h" |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 22 | |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 23 | #include "absl/memory/memory.h" |
Emircan Uysaler | 800787f | 2018-07-16 10:01:49 -0700 | [diff] [blame] | 24 | #include "api/video/color_space.h" |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 25 | #include "api/video/i010_buffer.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 26 | #include "common_video/include/video_frame_buffer.h" |
| 27 | #include "common_video/libyuv/include/webrtc_libyuv.h" |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 28 | #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h" |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 29 | #include "modules/video_coding/codecs/vp9/svc_rate_allocator.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "rtc_base/checks.h" |
| 31 | #include "rtc_base/keep_ref_until_done.h" |
| 32 | #include "rtc_base/logging.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 33 | #include "rtc_base/timeutils.h" |
| 34 | #include "rtc_base/trace_event.h" |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 35 | |
| 36 | namespace webrtc { |
| 37 | |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 38 | namespace { |
Marco | 6e89b25 | 2015-07-07 14:40:38 -0700 | [diff] [blame] | 39 | // Only positive speeds, range for real-time coding currently is: 5 - 8. |
| 40 | // Lower means slower/better quality, higher means fastest/lower quality. |
| 41 | int GetCpuSpeed(int width, int height) { |
Alex Glaznev | fecb7c3 | 2016-03-31 14:23:27 -0700 | [diff] [blame] | 42 | #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || defined(ANDROID) |
Marco | 002f0d0 | 2015-12-17 09:49:31 -0800 | [diff] [blame] | 43 | return 8; |
| 44 | #else |
Marco | 6e89b25 | 2015-07-07 14:40:38 -0700 | [diff] [blame] | 45 | // For smaller resolutions, use lower speed setting (get some coding gain at |
| 46 | // the cost of increased encoding complexity). |
| 47 | if (width * height <= 352 * 288) |
| 48 | return 5; |
| 49 | else |
| 50 | return 7; |
Marco | 002f0d0 | 2015-12-17 09:49:31 -0800 | [diff] [blame] | 51 | #endif |
Marco | 6e89b25 | 2015-07-07 14:40:38 -0700 | [diff] [blame] | 52 | } |
Emircan Uysaler | 715cc23 | 2018-07-23 17:50:43 -0700 | [diff] [blame] | 53 | // Helper class for extracting VP9 colorspace. |
| 54 | ColorSpace ExtractVP9ColorSpace(vpx_color_space_t space_t, |
| 55 | vpx_color_range_t range_t, |
| 56 | unsigned int bit_depth) { |
| 57 | ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid; |
| 58 | ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid; |
| 59 | ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid; |
| 60 | switch (space_t) { |
| 61 | case VPX_CS_BT_601: |
| 62 | case VPX_CS_SMPTE_170: |
| 63 | primaries = ColorSpace::PrimaryID::kSMPTE170M; |
| 64 | transfer = ColorSpace::TransferID::kSMPTE170M; |
| 65 | matrix = ColorSpace::MatrixID::kSMPTE170M; |
| 66 | break; |
| 67 | case VPX_CS_SMPTE_240: |
| 68 | primaries = ColorSpace::PrimaryID::kSMPTE240M; |
| 69 | transfer = ColorSpace::TransferID::kSMPTE240M; |
| 70 | matrix = ColorSpace::MatrixID::kSMPTE240M; |
| 71 | break; |
| 72 | case VPX_CS_BT_709: |
| 73 | primaries = ColorSpace::PrimaryID::kBT709; |
| 74 | transfer = ColorSpace::TransferID::kBT709; |
| 75 | matrix = ColorSpace::MatrixID::kBT709; |
| 76 | break; |
| 77 | case VPX_CS_BT_2020: |
| 78 | primaries = ColorSpace::PrimaryID::kBT2020; |
| 79 | switch (bit_depth) { |
| 80 | case 8: |
| 81 | transfer = ColorSpace::TransferID::kBT709; |
| 82 | break; |
| 83 | case 10: |
| 84 | transfer = ColorSpace::TransferID::kBT2020_10; |
| 85 | break; |
| 86 | default: |
| 87 | RTC_NOTREACHED(); |
| 88 | break; |
| 89 | } |
| 90 | matrix = ColorSpace::MatrixID::kBT2020_NCL; |
| 91 | break; |
| 92 | case VPX_CS_SRGB: |
| 93 | primaries = ColorSpace::PrimaryID::kBT709; |
| 94 | transfer = ColorSpace::TransferID::kIEC61966_2_1; |
| 95 | matrix = ColorSpace::MatrixID::kBT709; |
| 96 | break; |
| 97 | default: |
| 98 | break; |
| 99 | } |
| 100 | |
| 101 | ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid; |
| 102 | switch (range_t) { |
| 103 | case VPX_CR_STUDIO_RANGE: |
| 104 | range = ColorSpace::RangeID::kLimited; |
| 105 | break; |
| 106 | case VPX_CR_FULL_RANGE: |
| 107 | range = ColorSpace::RangeID::kFull; |
| 108 | break; |
| 109 | default: |
| 110 | break; |
| 111 | } |
| 112 | return ColorSpace(primaries, transfer, matrix, range); |
| 113 | } |
| 114 | } // namespace |
Marco | 6e89b25 | 2015-07-07 14:40:38 -0700 | [diff] [blame] | 115 | |
Emircan Uysaler | 98badbc | 2018-06-28 10:59:02 -0700 | [diff] [blame] | 116 | std::vector<SdpVideoFormat> SupportedVP9Codecs() { |
Max Morin | 221ee3c | 2018-09-14 07:50:07 +0000 | [diff] [blame] | 117 | // TODO(emircan): Add Profile 2 support after fixing browser_tests. |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 118 | std::vector<SdpVideoFormat> supported_formats{SdpVideoFormat( |
| 119 | cricket::kVp9CodecName, |
| 120 | {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})}; |
| 121 | return supported_formats; |
Peter Boström | 1299615 | 2016-05-14 02:03:18 +0200 | [diff] [blame] | 122 | } |
| 123 | |
Magnus Jedvert | 46a2765 | 2017-11-13 14:10:02 +0100 | [diff] [blame] | 124 | std::unique_ptr<VP9Encoder> VP9Encoder::Create() { |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 125 | return absl::make_unique<VP9EncoderImpl>(cricket::VideoCodec()); |
Emircan Uysaler | 98badbc | 2018-06-28 10:59:02 -0700 | [diff] [blame] | 126 | } |
| 127 | |
| 128 | std::unique_ptr<VP9Encoder> VP9Encoder::Create( |
| 129 | const cricket::VideoCodec& codec) { |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 130 | return absl::make_unique<VP9EncoderImpl>(codec); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 131 | } |
| 132 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 133 | void VP9EncoderImpl::EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt, |
| 134 | void* user_data) { |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 135 | VP9EncoderImpl* enc = static_cast<VP9EncoderImpl*>(user_data); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 136 | enc->GetEncodedLayerFrame(pkt); |
| 137 | } |
| 138 | |
Emircan Uysaler | 98badbc | 2018-06-28 10:59:02 -0700 | [diff] [blame] | 139 | VP9EncoderImpl::VP9EncoderImpl(const cricket::VideoCodec& codec) |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 140 | : encoded_image_(), |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 141 | encoded_complete_callback_(nullptr), |
Emircan Uysaler | 98badbc | 2018-06-28 10:59:02 -0700 | [diff] [blame] | 142 | profile_( |
| 143 | ParseSdpForVP9Profile(codec.params).value_or(VP9Profile::kProfile0)), |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 144 | inited_(false), |
| 145 | timestamp_(0), |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 146 | cpu_speed_(3), |
| 147 | rc_max_intra_target_(0), |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 148 | encoder_(nullptr), |
| 149 | config_(nullptr), |
| 150 | raw_(nullptr), |
| 151 | input_image_(nullptr), |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 152 | force_key_frame_(true), |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 153 | pics_since_key_(0), |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 154 | num_temporal_layers_(0), |
philipel | cfc319b | 2015-11-10 07:17:23 -0800 | [diff] [blame] | 155 | num_spatial_layers_(0), |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 156 | is_svc_(false), |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 157 | inter_layer_pred_(InterLayerPredMode::kOn), |
Sergey Silkin | be71a1e | 2018-05-17 16:46:43 +0200 | [diff] [blame] | 158 | is_flexible_mode_(false) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 159 | memset(&codec_, 0, sizeof(codec_)); |
johannkoenig | 8225c40 | 2017-01-26 13:23:44 -0800 | [diff] [blame] | 160 | memset(&svc_params_, 0, sizeof(vpx_svc_extra_cfg_t)); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 161 | } |
| 162 | |
| 163 | VP9EncoderImpl::~VP9EncoderImpl() { |
| 164 | Release(); |
| 165 | } |
| 166 | |
| 167 | int VP9EncoderImpl::Release() { |
Sergey Silkin | 3e871ea | 2018-03-02 13:11:04 +0100 | [diff] [blame] | 168 | int ret_val = WEBRTC_VIDEO_CODEC_OK; |
| 169 | |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 170 | if (encoded_image_._buffer != nullptr) { |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 171 | delete[] encoded_image_._buffer; |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 172 | encoded_image_._buffer = nullptr; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 173 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 174 | if (encoder_ != nullptr) { |
Sergey Silkin | 9039969 | 2018-03-02 14:44:10 +0100 | [diff] [blame] | 175 | if (inited_) { |
| 176 | if (vpx_codec_destroy(encoder_)) { |
| 177 | ret_val = WEBRTC_VIDEO_CODEC_MEMORY; |
| 178 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 179 | } |
| 180 | delete encoder_; |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 181 | encoder_ = nullptr; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 182 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 183 | if (config_ != nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 184 | delete config_; |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 185 | config_ = nullptr; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 186 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 187 | if (raw_ != nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 188 | vpx_img_free(raw_); |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 189 | raw_ = nullptr; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 190 | } |
| 191 | inited_ = false; |
Sergey Silkin | 3e871ea | 2018-03-02 13:11:04 +0100 | [diff] [blame] | 192 | return ret_val; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 193 | } |
| 194 | |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 195 | bool VP9EncoderImpl::ExplicitlyConfiguredSpatialLayers() const { |
| 196 | // We check target_bitrate_bps of the 0th layer to see if the spatial layers |
| 197 | // (i.e. bitrates) were explicitly configured. |
Sergey Silkin | beba1b2 | 2018-09-11 11:40:25 +0200 | [diff] [blame] | 198 | return codec_.spatialLayers[0].targetBitrate > 0; |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 199 | } |
| 200 | |
Erik Språng | 566124a | 2018-04-23 12:32:22 +0200 | [diff] [blame] | 201 | bool VP9EncoderImpl::SetSvcRates( |
| 202 | const VideoBitrateAllocation& bitrate_allocation) { |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 203 | config_->rc_target_bitrate = bitrate_allocation.get_sum_kbps(); |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 204 | |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 205 | if (ExplicitlyConfiguredSpatialLayers()) { |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 206 | for (size_t sl_idx = 0; sl_idx < num_spatial_layers_; ++sl_idx) { |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 207 | const bool was_layer_enabled = (config_->ss_target_bitrate[sl_idx] > 0); |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 208 | config_->ss_target_bitrate[sl_idx] = |
| 209 | bitrate_allocation.GetSpatialLayerSum(sl_idx) / 1000; |
| 210 | |
| 211 | for (size_t tl_idx = 0; tl_idx < num_temporal_layers_; ++tl_idx) { |
| 212 | config_->layer_target_bitrate[sl_idx * num_temporal_layers_ + tl_idx] = |
| 213 | bitrate_allocation.GetTemporalLayerSum(sl_idx, tl_idx) / 1000; |
| 214 | } |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 215 | |
| 216 | const bool is_layer_enabled = (config_->ss_target_bitrate[sl_idx] > 0); |
| 217 | if (is_layer_enabled && !was_layer_enabled) { |
| 218 | if (inter_layer_pred_ == InterLayerPredMode::kOff || |
| 219 | inter_layer_pred_ == InterLayerPredMode::kOnKeyPic) { |
| 220 | // TODO(wemb:1526): remove key frame request when issue is fixed. |
| 221 | force_key_frame_ = true; |
| 222 | } |
| 223 | } |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 224 | |
| 225 | if (!was_layer_enabled) { |
| 226 | // Reset frame rate controller if layer is resumed after pause. |
| 227 | framerate_controller_[sl_idx].Reset(); |
| 228 | } |
| 229 | |
| 230 | framerate_controller_[sl_idx].SetTargetRate( |
Sergey Silkin | beba1b2 | 2018-09-11 11:40:25 +0200 | [diff] [blame] | 231 | std::min(static_cast<float>(codec_.maxFramerate), |
| 232 | codec_.spatialLayers[sl_idx].maxFramerate)); |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 233 | } |
| 234 | } else { |
| 235 | float rate_ratio[VPX_MAX_LAYERS] = {0}; |
| 236 | float total = 0; |
Sergey Silkin | 908689d | 2018-08-20 09:28:45 +0200 | [diff] [blame] | 237 | for (int i = 0; i < num_spatial_layers_; ++i) { |
johannkoenig | 8225c40 | 2017-01-26 13:23:44 -0800 | [diff] [blame] | 238 | if (svc_params_.scaling_factor_num[i] <= 0 || |
| 239 | svc_params_.scaling_factor_den[i] <= 0) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 240 | RTC_LOG(LS_ERROR) << "Scaling factors not specified!"; |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 241 | return false; |
| 242 | } |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 243 | rate_ratio[i] = static_cast<float>(svc_params_.scaling_factor_num[i]) / |
| 244 | svc_params_.scaling_factor_den[i]; |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 245 | total += rate_ratio[i]; |
| 246 | } |
| 247 | |
Sergey Silkin | 908689d | 2018-08-20 09:28:45 +0200 | [diff] [blame] | 248 | for (int i = 0; i < num_spatial_layers_; ++i) { |
Rasmus Brandt | 58cd385 | 2018-06-26 13:41:16 +0200 | [diff] [blame] | 249 | RTC_CHECK_GT(total, 0); |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 250 | config_->ss_target_bitrate[i] = static_cast<unsigned int>( |
| 251 | config_->rc_target_bitrate * rate_ratio[i] / total); |
| 252 | if (num_temporal_layers_ == 1) { |
| 253 | config_->layer_target_bitrate[i] = config_->ss_target_bitrate[i]; |
| 254 | } else if (num_temporal_layers_ == 2) { |
| 255 | config_->layer_target_bitrate[i * num_temporal_layers_] = |
| 256 | config_->ss_target_bitrate[i] * 2 / 3; |
| 257 | config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = |
| 258 | config_->ss_target_bitrate[i]; |
| 259 | } else if (num_temporal_layers_ == 3) { |
| 260 | config_->layer_target_bitrate[i * num_temporal_layers_] = |
| 261 | config_->ss_target_bitrate[i] / 2; |
| 262 | config_->layer_target_bitrate[i * num_temporal_layers_ + 1] = |
| 263 | config_->layer_target_bitrate[i * num_temporal_layers_] + |
| 264 | (config_->ss_target_bitrate[i] / 4); |
| 265 | config_->layer_target_bitrate[i * num_temporal_layers_ + 2] = |
| 266 | config_->ss_target_bitrate[i]; |
| 267 | } else { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 268 | RTC_LOG(LS_ERROR) << "Unsupported number of temporal layers: " |
| 269 | << num_temporal_layers_; |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 270 | return false; |
| 271 | } |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 272 | |
| 273 | framerate_controller_[i].SetTargetRate(codec_.maxFramerate); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 274 | } |
| 275 | } |
Sergey Silkin | 908689d | 2018-08-20 09:28:45 +0200 | [diff] [blame] | 276 | |
| 277 | num_active_spatial_layers_ = 0; |
| 278 | for (int i = 0; i < num_spatial_layers_; ++i) { |
| 279 | if (config_->ss_target_bitrate[i] > 0) { |
| 280 | ++num_active_spatial_layers_; |
| 281 | } |
| 282 | } |
| 283 | RTC_DCHECK_GT(num_active_spatial_layers_, 0); |
| 284 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 285 | return true; |
| 286 | } |
| 287 | |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 288 | int VP9EncoderImpl::SetRateAllocation( |
Erik Språng | 566124a | 2018-04-23 12:32:22 +0200 | [diff] [blame] | 289 | const VideoBitrateAllocation& bitrate_allocation, |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 290 | uint32_t frame_rate) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 291 | if (!inited_) { |
| 292 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 293 | } |
| 294 | if (encoder_->err) { |
| 295 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 296 | } |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 297 | if (frame_rate < 1) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 298 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 299 | } |
| 300 | // Update bit rate |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 301 | if (codec_.maxBitrate > 0 && |
| 302 | bitrate_allocation.get_sum_kbps() > codec_.maxBitrate) { |
| 303 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 304 | } |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 305 | |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 306 | codec_.maxFramerate = frame_rate; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 307 | |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 308 | if (!SetSvcRates(bitrate_allocation)) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 309 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 310 | } |
| 311 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 312 | // Update encoder context |
| 313 | if (vpx_codec_enc_config_set(encoder_, config_)) { |
| 314 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 315 | } |
| 316 | return WEBRTC_VIDEO_CODEC_OK; |
| 317 | } |
| 318 | |
| 319 | int VP9EncoderImpl::InitEncode(const VideoCodec* inst, |
| 320 | int number_of_cores, |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 321 | size_t /*max_payload_size*/) { |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 322 | if (inst == nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 323 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 324 | } |
| 325 | if (inst->maxFramerate < 1) { |
| 326 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 327 | } |
| 328 | // Allow zero to represent an unspecified maxBitRate |
| 329 | if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) { |
| 330 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 331 | } |
| 332 | if (inst->width < 1 || inst->height < 1) { |
| 333 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 334 | } |
| 335 | if (number_of_cores < 1) { |
| 336 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 337 | } |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 338 | if (inst->VP9().numberOfTemporalLayers > 3) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 339 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 340 | } |
ilnik | 2a8c2f5 | 2017-02-15 02:23:28 -0800 | [diff] [blame] | 341 | // libvpx probably does not support more than 3 spatial layers. |
| 342 | if (inst->VP9().numberOfSpatialLayers > 3) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 343 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 344 | } |
philipel | cfc319b | 2015-11-10 07:17:23 -0800 | [diff] [blame] | 345 | |
asapersson | 86956de | 2016-01-26 01:05:20 -0800 | [diff] [blame] | 346 | int ret_val = Release(); |
| 347 | if (ret_val < 0) { |
| 348 | return ret_val; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 349 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 350 | if (encoder_ == nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 351 | encoder_ = new vpx_codec_ctx_t; |
| 352 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 353 | if (config_ == nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 354 | config_ = new vpx_codec_enc_cfg_t; |
| 355 | } |
| 356 | timestamp_ = 0; |
| 357 | if (&codec_ != inst) { |
| 358 | codec_ = *inst; |
| 359 | } |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 360 | |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 361 | num_spatial_layers_ = inst->VP9().numberOfSpatialLayers; |
Niels Möller | 65fb404 | 2018-04-25 14:46:06 +0200 | [diff] [blame] | 362 | RTC_DCHECK_GT(num_spatial_layers_, 0); |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 363 | num_temporal_layers_ = inst->VP9().numberOfTemporalLayers; |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 364 | if (num_temporal_layers_ == 0) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 365 | num_temporal_layers_ = 1; |
Sergey Silkin | 042661b | 2018-09-10 10:39:35 +0000 | [diff] [blame] | 366 | } |
Sergey Silkin | ae9e188 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 367 | |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 368 | framerate_controller_ = std::vector<FramerateController>( |
| 369 | num_spatial_layers_, FramerateController(codec_.maxFramerate)); |
| 370 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 371 | is_svc_ = (num_spatial_layers_ > 1 || num_temporal_layers_ > 1); |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 372 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 373 | // Allocate memory for encoded image |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 374 | if (encoded_image_._buffer != nullptr) { |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 375 | delete[] encoded_image_._buffer; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 376 | } |
nisse | eb44b39 | 2017-04-28 07:18:05 -0700 | [diff] [blame] | 377 | encoded_image_._size = |
| 378 | CalcBufferSize(VideoType::kI420, codec_.width, codec_.height); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 379 | encoded_image_._buffer = new uint8_t[encoded_image_._size]; |
| 380 | encoded_image_._completeFrame = true; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 381 | // Populate encoder configuration with default values. |
| 382 | if (vpx_codec_enc_config_default(vpx_codec_vp9_cx(), config_, 0)) { |
| 383 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 384 | } |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 385 | |
| 386 | vpx_img_fmt img_fmt = VPX_IMG_FMT_NONE; |
| 387 | unsigned int bits_for_storage = 8; |
| 388 | switch (profile_) { |
| 389 | case VP9Profile::kProfile0: |
| 390 | img_fmt = VPX_IMG_FMT_I420; |
| 391 | bits_for_storage = 8; |
| 392 | config_->g_bit_depth = VPX_BITS_8; |
| 393 | config_->g_profile = 0; |
| 394 | config_->g_input_bit_depth = 8; |
| 395 | break; |
| 396 | case VP9Profile::kProfile2: |
| 397 | img_fmt = VPX_IMG_FMT_I42016; |
| 398 | bits_for_storage = 16; |
| 399 | config_->g_bit_depth = VPX_BITS_10; |
| 400 | config_->g_profile = 2; |
| 401 | config_->g_input_bit_depth = 10; |
| 402 | break; |
| 403 | } |
| 404 | |
| 405 | // Creating a wrapper to the image - setting image data to nullptr. Actual |
| 406 | // pointer will be set in encode. Setting align to 1, as it is meaningless |
| 407 | // (actual memory is not allocated). |
| 408 | raw_ = |
| 409 | vpx_img_wrap(nullptr, img_fmt, codec_.width, codec_.height, 1, nullptr); |
| 410 | raw_->bit_depth = bits_for_storage; |
| 411 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 412 | config_->g_w = codec_.width; |
| 413 | config_->g_h = codec_.height; |
| 414 | config_->rc_target_bitrate = inst->startBitrate; // in kbit/s |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 415 | config_->g_error_resilient = is_svc_ ? VPX_ERROR_RESILIENT_DEFAULT : 0; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 416 | // Setting the time base of the codec. |
| 417 | config_->g_timebase.num = 1; |
| 418 | config_->g_timebase.den = 90000; |
| 419 | config_->g_lag_in_frames = 0; // 0- no frame lagging |
| 420 | config_->g_threads = 1; |
| 421 | // Rate control settings. |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 422 | config_->rc_dropframe_thresh = inst->VP9().frameDroppingOn ? 30 : 0; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 423 | config_->rc_end_usage = VPX_CBR; |
| 424 | config_->g_pass = VPX_RC_ONE_PASS; |
| 425 | config_->rc_min_quantizer = 2; |
marpan@webrtc.org | dc8a9da | 2015-01-27 23:08:24 +0000 | [diff] [blame] | 426 | config_->rc_max_quantizer = 52; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 427 | config_->rc_undershoot_pct = 50; |
| 428 | config_->rc_overshoot_pct = 50; |
| 429 | config_->rc_buf_initial_sz = 500; |
| 430 | config_->rc_buf_optimal_sz = 600; |
| 431 | config_->rc_buf_sz = 1000; |
| 432 | // Set the maximum target size of any key-frame. |
| 433 | rc_max_intra_target_ = MaxIntraTarget(config_->rc_buf_optimal_sz); |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 434 | if (inst->VP9().keyFrameInterval > 0) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 435 | config_->kf_mode = VPX_KF_AUTO; |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 436 | config_->kf_max_dist = inst->VP9().keyFrameInterval; |
Åsa Persson | ff24c04 | 2015-12-04 10:58:08 +0100 | [diff] [blame] | 437 | // Needs to be set (in svc mode) to get correct periodic key frame interval |
| 438 | // (will have no effect in non-svc). |
| 439 | config_->kf_min_dist = config_->kf_max_dist; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 440 | } else { |
| 441 | config_->kf_mode = VPX_KF_DISABLED; |
| 442 | } |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 443 | config_->rc_resize_allowed = inst->VP9().automaticResizeOn ? 1 : 0; |
marpan@webrtc.org | 38d11b8 | 2015-01-26 15:21:36 +0000 | [diff] [blame] | 444 | // Determine number of threads based on the image size and #cores. |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 445 | config_->g_threads = |
| 446 | NumberOfThreads(config_->g_w, config_->g_h, number_of_cores); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 447 | |
Marco | 6e89b25 | 2015-07-07 14:40:38 -0700 | [diff] [blame] | 448 | cpu_speed_ = GetCpuSpeed(config_->g_w, config_->g_h); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 449 | |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 450 | is_flexible_mode_ = inst->VP9().flexibleMode; |
Sergey Silkin | be71a1e | 2018-05-17 16:46:43 +0200 | [diff] [blame] | 451 | |
Sergey Silkin | be71a1e | 2018-05-17 16:46:43 +0200 | [diff] [blame] | 452 | if (num_temporal_layers_ == 1) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 453 | gof_.SetGofInfoVP9(kTemporalStructureMode1); |
| 454 | config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING; |
| 455 | config_->ts_number_layers = 1; |
| 456 | config_->ts_rate_decimator[0] = 1; |
| 457 | config_->ts_periodicity = 1; |
| 458 | config_->ts_layer_id[0] = 0; |
| 459 | } else if (num_temporal_layers_ == 2) { |
| 460 | gof_.SetGofInfoVP9(kTemporalStructureMode2); |
| 461 | config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0101; |
| 462 | config_->ts_number_layers = 2; |
| 463 | config_->ts_rate_decimator[0] = 2; |
| 464 | config_->ts_rate_decimator[1] = 1; |
| 465 | config_->ts_periodicity = 2; |
| 466 | config_->ts_layer_id[0] = 0; |
| 467 | config_->ts_layer_id[1] = 1; |
| 468 | } else if (num_temporal_layers_ == 3) { |
| 469 | gof_.SetGofInfoVP9(kTemporalStructureMode3); |
| 470 | config_->temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_0212; |
| 471 | config_->ts_number_layers = 3; |
| 472 | config_->ts_rate_decimator[0] = 4; |
| 473 | config_->ts_rate_decimator[1] = 2; |
| 474 | config_->ts_rate_decimator[2] = 1; |
| 475 | config_->ts_periodicity = 4; |
| 476 | config_->ts_layer_id[0] = 0; |
| 477 | config_->ts_layer_id[1] = 2; |
| 478 | config_->ts_layer_id[2] = 1; |
| 479 | config_->ts_layer_id[3] = 2; |
| 480 | } else { |
| 481 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 482 | } |
| 483 | |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 484 | inter_layer_pred_ = inst->VP9().interLayerPred; |
| 485 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 486 | ref_buf_.clear(); |
| 487 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 488 | return InitAndSetControlSettings(inst); |
| 489 | } |
| 490 | |
marpan@webrtc.org | 38d11b8 | 2015-01-26 15:21:36 +0000 | [diff] [blame] | 491 | int VP9EncoderImpl::NumberOfThreads(int width, |
| 492 | int height, |
| 493 | int number_of_cores) { |
| 494 | // Keep the number of encoder threads equal to the possible number of column |
| 495 | // tiles, which is (1, 2, 4, 8). See comments below for VP9E_SET_TILE_COLUMNS. |
| 496 | if (width * height >= 1280 * 720 && number_of_cores > 4) { |
| 497 | return 4; |
jianj | 23173a3 | 2017-07-12 16:11:09 -0700 | [diff] [blame] | 498 | } else if (width * height >= 640 * 360 && number_of_cores > 2) { |
marpan@webrtc.org | 38d11b8 | 2015-01-26 15:21:36 +0000 | [diff] [blame] | 499 | return 2; |
| 500 | } else { |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 501 | // Use 2 threads for low res on ARM. |
Jerome Jiang | 831af37 | 2017-12-05 10:44:35 -0800 | [diff] [blame] | 502 | #if defined(WEBRTC_ARCH_ARM) || defined(WEBRTC_ARCH_ARM64) || \ |
| 503 | defined(WEBRTC_ANDROID) |
| 504 | if (width * height >= 320 * 180 && number_of_cores > 2) { |
| 505 | return 2; |
| 506 | } |
| 507 | #endif |
marpan@webrtc.org | 38d11b8 | 2015-01-26 15:21:36 +0000 | [diff] [blame] | 508 | // 1 thread less than VGA. |
| 509 | return 1; |
| 510 | } |
| 511 | } |
| 512 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 513 | int VP9EncoderImpl::InitAndSetControlSettings(const VideoCodec* inst) { |
Åsa Persson | ff24c04 | 2015-12-04 10:58:08 +0100 | [diff] [blame] | 514 | // Set QP-min/max per spatial and temporal layer. |
| 515 | int tot_num_layers = num_spatial_layers_ * num_temporal_layers_; |
| 516 | for (int i = 0; i < tot_num_layers; ++i) { |
johannkoenig | 8225c40 | 2017-01-26 13:23:44 -0800 | [diff] [blame] | 517 | svc_params_.max_quantizers[i] = config_->rc_max_quantizer; |
| 518 | svc_params_.min_quantizers[i] = config_->rc_min_quantizer; |
Åsa Persson | ff24c04 | 2015-12-04 10:58:08 +0100 | [diff] [blame] | 519 | } |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 520 | config_->ss_number_layers = num_spatial_layers_; |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 521 | if (ExplicitlyConfiguredSpatialLayers()) { |
| 522 | for (int i = 0; i < num_spatial_layers_; ++i) { |
| 523 | const auto& layer = codec_.spatialLayers[i]; |
Rasmus Brandt | 58cd385 | 2018-06-26 13:41:16 +0200 | [diff] [blame] | 524 | RTC_CHECK_GT(layer.width, 0); |
Sergey Silkin | 13e7434 | 2018-03-02 12:28:00 +0100 | [diff] [blame] | 525 | const int scale_factor = codec_.width / layer.width; |
| 526 | RTC_DCHECK_GT(scale_factor, 0); |
| 527 | |
| 528 | // Ensure scaler factor is integer. |
| 529 | if (scale_factor * layer.width != codec_.width) { |
| 530 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 531 | } |
| 532 | |
| 533 | // Ensure scale factor is the same in both dimensions. |
| 534 | if (scale_factor * layer.height != codec_.height) { |
| 535 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 536 | } |
| 537 | |
| 538 | // Ensure scale factor is power of two. |
| 539 | const bool is_pow_of_two = (scale_factor & (scale_factor - 1)) == 0; |
| 540 | if (!is_pow_of_two) { |
| 541 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 542 | } |
| 543 | |
| 544 | svc_params_.scaling_factor_num[i] = 1; |
| 545 | svc_params_.scaling_factor_den[i] = scale_factor; |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 546 | |
| 547 | RTC_DCHECK_GT(codec_.spatialLayers[i].maxFramerate, 0); |
| 548 | RTC_DCHECK_LE(codec_.spatialLayers[i].maxFramerate, codec_.maxFramerate); |
| 549 | if (i > 0) { |
| 550 | // Frame rate of high spatial layer is supposed to be equal or higher |
| 551 | // than frame rate of low spatial layer. |
| 552 | RTC_DCHECK_GE(codec_.spatialLayers[i].maxFramerate, |
| 553 | codec_.spatialLayers[i - 1].maxFramerate); |
| 554 | } |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 555 | } |
| 556 | } else { |
| 557 | int scaling_factor_num = 256; |
| 558 | for (int i = num_spatial_layers_ - 1; i >= 0; --i) { |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 559 | // 1:2 scaling in each dimension. |
johannkoenig | 8225c40 | 2017-01-26 13:23:44 -0800 | [diff] [blame] | 560 | svc_params_.scaling_factor_num[i] = scaling_factor_num; |
| 561 | svc_params_.scaling_factor_den[i] = 256; |
sprang | ce4aef1 | 2015-11-02 07:23:20 -0800 | [diff] [blame] | 562 | } |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 563 | } |
| 564 | |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 565 | SvcRateAllocator init_allocator(codec_); |
Erik Språng | 566124a | 2018-04-23 12:32:22 +0200 | [diff] [blame] | 566 | VideoBitrateAllocation allocation = init_allocator.GetAllocation( |
Sergey Silkin | 8668496 | 2018-03-28 19:32:37 +0200 | [diff] [blame] | 567 | inst->startBitrate * 1000, inst->maxFramerate); |
| 568 | if (!SetSvcRates(allocation)) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 569 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 570 | } |
| 571 | |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 572 | const vpx_codec_err_t rv = vpx_codec_enc_init( |
| 573 | encoder_, vpx_codec_vp9_cx(), config_, |
| 574 | config_->g_bit_depth == VPX_BITS_8 ? 0 : VPX_CODEC_USE_HIGHBITDEPTH); |
| 575 | if (rv != VPX_CODEC_OK) { |
| 576 | RTC_LOG(LS_ERROR) << "Init error: " << vpx_codec_err_to_string(rv); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 577 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 578 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 579 | vpx_codec_control(encoder_, VP8E_SET_CPUUSED, cpu_speed_); |
| 580 | vpx_codec_control(encoder_, VP8E_SET_MAX_INTRA_BITRATE_PCT, |
| 581 | rc_max_intra_target_); |
| 582 | vpx_codec_control(encoder_, VP9E_SET_AQ_MODE, |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 583 | inst->VP9().adaptiveQpMode ? 3 : 0); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 584 | |
jianj | 822e593 | 2017-07-12 16:09:58 -0700 | [diff] [blame] | 585 | vpx_codec_control(encoder_, VP9E_SET_FRAME_PARALLEL_DECODING, 0); |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 586 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 587 | if (is_svc_) { |
| 588 | vpx_codec_control(encoder_, VP9E_SET_SVC, 1); |
| 589 | vpx_codec_control(encoder_, VP9E_SET_SVC_PARAMETERS, &svc_params_); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 590 | } |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 591 | |
| 592 | if (num_spatial_layers_ > 1) { |
| 593 | switch (inter_layer_pred_) { |
| 594 | case InterLayerPredMode::kOn: |
| 595 | vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 0); |
| 596 | break; |
| 597 | case InterLayerPredMode::kOff: |
| 598 | vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 1); |
| 599 | break; |
| 600 | case InterLayerPredMode::kOnKeyPic: |
| 601 | vpx_codec_control(encoder_, VP9E_SET_SVC_INTER_LAYER_PRED, 2); |
| 602 | break; |
| 603 | default: |
| 604 | RTC_NOTREACHED(); |
| 605 | } |
Sergey Silkin | be71a1e | 2018-05-17 16:46:43 +0200 | [diff] [blame] | 606 | |
Sergey Silkin | ee20336 | 2018-05-30 11:34:08 +0200 | [diff] [blame] | 607 | // Configure encoder to drop entire superframe whenever it needs to drop |
| 608 | // a layer. This mode is prefered over per-layer dropping which causes |
| 609 | // quality flickering and is not compatible with RTP non-flexible mode. |
| 610 | vpx_svc_frame_drop_t svc_drop_frame; |
| 611 | memset(&svc_drop_frame, 0, sizeof(svc_drop_frame)); |
| 612 | svc_drop_frame.framedrop_mode = FULL_SUPERFRAME_DROP; |
Sergey Silkin | d45b345 | 2018-05-31 16:00:24 +0200 | [diff] [blame] | 613 | svc_drop_frame.max_consec_drop = std::numeric_limits<int>::max(); |
Sergey Silkin | ee20336 | 2018-05-30 11:34:08 +0200 | [diff] [blame] | 614 | for (size_t i = 0; i < num_spatial_layers_; ++i) { |
| 615 | svc_drop_frame.framedrop_thresh[i] = config_->rc_dropframe_thresh; |
Sergey Silkin | be71a1e | 2018-05-17 16:46:43 +0200 | [diff] [blame] | 616 | } |
Sergey Silkin | ee20336 | 2018-05-30 11:34:08 +0200 | [diff] [blame] | 617 | vpx_codec_control(encoder_, VP9E_SET_SVC_FRAME_DROP_LAYER, &svc_drop_frame); |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 618 | } |
| 619 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 620 | // Register callback for getting each spatial layer. |
| 621 | vpx_codec_priv_output_cx_pkt_cb_pair_t cbp = { |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 622 | VP9EncoderImpl::EncoderOutputCodedPacketCallback, |
| 623 | reinterpret_cast<void*>(this)}; |
| 624 | vpx_codec_control(encoder_, VP9E_REGISTER_CX_CALLBACK, |
| 625 | reinterpret_cast<void*>(&cbp)); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 626 | |
marpan@webrtc.org | 38d11b8 | 2015-01-26 15:21:36 +0000 | [diff] [blame] | 627 | // Control function to set the number of column tiles in encoding a frame, in |
| 628 | // log2 unit: e.g., 0 = 1 tile column, 1 = 2 tile columns, 2 = 4 tile columns. |
| 629 | // The number tile columns will be capped by the encoder based on image size |
| 630 | // (minimum width of tile column is 256 pixels, maximum is 4096). |
| 631 | vpx_codec_control(encoder_, VP9E_SET_TILE_COLUMNS, (config_->g_threads >> 1)); |
jianj | cb5d115 | 2017-03-28 23:56:08 -0700 | [diff] [blame] | 632 | |
| 633 | // Turn on row-based multithreading. |
| 634 | vpx_codec_control(encoder_, VP9E_SET_ROW_MT, 1); |
jianj | 6bf57e3 | 2017-06-05 13:43:49 -0700 | [diff] [blame] | 635 | |
Alex Glaznev | fecb7c3 | 2016-03-31 14:23:27 -0700 | [diff] [blame] | 636 | #if !defined(WEBRTC_ARCH_ARM) && !defined(WEBRTC_ARCH_ARM64) && \ |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 637 | !defined(ANDROID) |
jianj | 6bf57e3 | 2017-06-05 13:43:49 -0700 | [diff] [blame] | 638 | // Do not enable the denoiser on ARM since optimization is pending. |
| 639 | // Denoiser is on by default on other platforms. |
marpan@webrtc.org | 16a87b9 | 2015-03-05 22:19:00 +0000 | [diff] [blame] | 640 | vpx_codec_control(encoder_, VP9E_SET_NOISE_SENSITIVITY, |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 641 | inst->VP9().denoisingOn ? 1 : 0); |
marpan@webrtc.org | 16a87b9 | 2015-03-05 22:19:00 +0000 | [diff] [blame] | 642 | #endif |
jianj | 6bf57e3 | 2017-06-05 13:43:49 -0700 | [diff] [blame] | 643 | |
Niels Möller | e3cf3d0 | 2018-06-13 11:52:16 +0200 | [diff] [blame] | 644 | if (codec_.mode == VideoCodecMode::kScreensharing) { |
ivica | 242d638 | 2015-09-04 06:13:23 -0700 | [diff] [blame] | 645 | // Adjust internal parameters to screen content. |
| 646 | vpx_codec_control(encoder_, VP9E_SET_TUNE_CONTENT, 1); |
ivica | 242d638 | 2015-09-04 06:13:23 -0700 | [diff] [blame] | 647 | } |
Marco | 2520e72 | 2015-09-16 14:05:00 -0700 | [diff] [blame] | 648 | // Enable encoder skip of static/low content blocks. |
| 649 | vpx_codec_control(encoder_, VP8E_SET_STATIC_THRESHOLD, 1); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 650 | inited_ = true; |
| 651 | return WEBRTC_VIDEO_CODEC_OK; |
| 652 | } |
| 653 | |
| 654 | uint32_t VP9EncoderImpl::MaxIntraTarget(uint32_t optimal_buffer_size) { |
| 655 | // Set max to the optimal buffer level (normalized by target BR), |
| 656 | // and scaled by a scale_par. |
| 657 | // Max target size = scale_par * optimal_buffer_size * targetBR[Kbps]. |
| 658 | // This value is presented in percentage of perFrameBw: |
| 659 | // perFrameBw = targetBR[Kbps] * 1000 / framerate. |
| 660 | // The target in % is as follows: |
| 661 | float scale_par = 0.5; |
| 662 | uint32_t target_pct = |
| 663 | optimal_buffer_size * scale_par * codec_.maxFramerate / 10; |
| 664 | // Don't go below 3 times the per frame bandwidth. |
| 665 | const uint32_t min_intra_size = 300; |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 666 | return (target_pct < min_intra_size) ? min_intra_size : target_pct; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 667 | } |
| 668 | |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 669 | int VP9EncoderImpl::Encode(const VideoFrame& input_image, |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 670 | const CodecSpecificInfo* codec_specific_info, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 671 | const std::vector<FrameType>* frame_types) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 672 | if (!inited_) { |
| 673 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 674 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 675 | if (encoded_complete_callback_ == nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 676 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 677 | } |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 678 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 679 | // We only support one stream at the moment. |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 680 | if (frame_types && !frame_types->empty()) { |
| 681 | if ((*frame_types)[0] == kVideoFrameKey) { |
| 682 | force_key_frame_ = true; |
| 683 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 684 | } |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 685 | |
Niels Möller | e3cf3d0 | 2018-06-13 11:52:16 +0200 | [diff] [blame] | 686 | if (VideoCodecMode::kScreensharing == codec_.mode && !force_key_frame_) { |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 687 | // Skip encoding spatial layer frames if their target frame rate is lower |
| 688 | // than actual input frame rate. |
| 689 | vpx_svc_layer_id_t layer_id = {0}; |
| 690 | const size_t gof_idx = (pics_since_key_ + 1) % gof_.num_frames_in_gof; |
| 691 | layer_id.temporal_layer_id = gof_.temporal_idx[gof_idx]; |
| 692 | |
| 693 | const uint32_t frame_timestamp_ms = |
| 694 | 1000 * input_image.timestamp() / kVideoPayloadTypeFrequency; |
| 695 | |
| 696 | for (uint8_t sl_idx = 0; sl_idx < num_active_spatial_layers_; ++sl_idx) { |
| 697 | if (framerate_controller_[sl_idx].DropFrame(frame_timestamp_ms)) { |
| 698 | ++layer_id.spatial_layer_id; |
| 699 | } else { |
| 700 | break; |
| 701 | } |
| 702 | } |
| 703 | |
| 704 | RTC_DCHECK_LE(layer_id.spatial_layer_id, num_active_spatial_layers_); |
| 705 | if (layer_id.spatial_layer_id >= num_active_spatial_layers_) { |
| 706 | // Drop entire picture. |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 707 | return WEBRTC_VIDEO_CODEC_OK; |
| 708 | } |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 709 | |
| 710 | vpx_codec_control(encoder_, VP9E_SET_SVC_LAYER_ID, &layer_id); |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 711 | } |
| 712 | |
kwiberg | 352444f | 2016-11-28 15:58:53 -0800 | [diff] [blame] | 713 | RTC_DCHECK_EQ(input_image.width(), raw_->d_w); |
| 714 | RTC_DCHECK_EQ(input_image.height(), raw_->d_h); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 715 | |
| 716 | // Set input image for use in the callback. |
| 717 | // This was necessary since you need some information from input_image. |
| 718 | // You can save only the necessary information (such as timestamp) instead of |
| 719 | // doing this. |
| 720 | input_image_ = &input_image; |
| 721 | |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 722 | // Keep reference to buffer until encode completes. |
| 723 | rtc::scoped_refptr<I420BufferInterface> i420_buffer; |
| 724 | rtc::scoped_refptr<I010BufferInterface> i010_buffer; |
| 725 | switch (profile_) { |
| 726 | case VP9Profile::kProfile0: { |
| 727 | i420_buffer = input_image.video_frame_buffer()->ToI420(); |
| 728 | // Image in vpx_image_t format. |
| 729 | // Input image is const. VPX's raw image is not defined as const. |
| 730 | raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>(i420_buffer->DataY()); |
| 731 | raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>(i420_buffer->DataU()); |
| 732 | raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>(i420_buffer->DataV()); |
| 733 | raw_->stride[VPX_PLANE_Y] = i420_buffer->StrideY(); |
| 734 | raw_->stride[VPX_PLANE_U] = i420_buffer->StrideU(); |
| 735 | raw_->stride[VPX_PLANE_V] = i420_buffer->StrideV(); |
| 736 | break; |
| 737 | } |
| 738 | case VP9Profile::kProfile2: { |
| 739 | // We can inject kI010 frames directly for encode. All other formats |
| 740 | // should be converted to it. |
| 741 | switch (input_image.video_frame_buffer()->type()) { |
| 742 | case VideoFrameBuffer::Type::kI010: { |
| 743 | i010_buffer = input_image.video_frame_buffer()->GetI010(); |
| 744 | break; |
| 745 | } |
| 746 | default: { |
| 747 | i010_buffer = |
| 748 | I010Buffer::Copy(*input_image.video_frame_buffer()->ToI420()); |
| 749 | } |
| 750 | } |
| 751 | raw_->planes[VPX_PLANE_Y] = const_cast<uint8_t*>( |
| 752 | reinterpret_cast<const uint8_t*>(i010_buffer->DataY())); |
| 753 | raw_->planes[VPX_PLANE_U] = const_cast<uint8_t*>( |
| 754 | reinterpret_cast<const uint8_t*>(i010_buffer->DataU())); |
| 755 | raw_->planes[VPX_PLANE_V] = const_cast<uint8_t*>( |
| 756 | reinterpret_cast<const uint8_t*>(i010_buffer->DataV())); |
| 757 | raw_->stride[VPX_PLANE_Y] = i010_buffer->StrideY() * 2; |
| 758 | raw_->stride[VPX_PLANE_U] = i010_buffer->StrideU() * 2; |
| 759 | raw_->stride[VPX_PLANE_V] = i010_buffer->StrideV() * 2; |
| 760 | break; |
| 761 | } |
| 762 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 763 | |
philipel | cfc319b | 2015-11-10 07:17:23 -0800 | [diff] [blame] | 764 | vpx_enc_frame_flags_t flags = 0; |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 765 | if (force_key_frame_) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 766 | flags = VPX_EFLAG_FORCE_KF; |
| 767 | } |
philipel | cfc319b | 2015-11-10 07:17:23 -0800 | [diff] [blame] | 768 | |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 769 | // TODO(ssilkin): Frame duration should be specified per spatial layer |
| 770 | // since their frame rate can be different. For now calculate frame duration |
| 771 | // based on target frame rate of the highest spatial layer, which frame rate |
| 772 | // is supposed to be equal or higher than frame rate of low spatial layers. |
| 773 | // Also, timestamp should represent actual time passed since previous frame |
| 774 | // (not 'expected' time). Then rate controller can drain buffer more |
| 775 | // accurately. |
| 776 | RTC_DCHECK_GE(framerate_controller_.size(), num_active_spatial_layers_); |
Sergey Silkin | f87bb46 | 2018-09-17 09:37:37 +0200 | [diff] [blame^] | 777 | float target_framerate_fps = |
| 778 | (codec_.mode == VideoCodecMode::kScreensharing) |
| 779 | ? framerate_controller_[num_active_spatial_layers_ - 1] |
| 780 | .GetTargetRate() |
| 781 | : codec_.maxFramerate; |
| 782 | uint32_t duration = static_cast<uint32_t>(90000 / target_framerate_fps); |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 783 | const vpx_codec_err_t rv = vpx_codec_encode(encoder_, raw_, timestamp_, |
| 784 | duration, flags, VPX_DL_REALTIME); |
| 785 | if (rv != VPX_CODEC_OK) { |
| 786 | RTC_LOG(LS_ERROR) << "Encoding error: " << vpx_codec_err_to_string(rv) |
| 787 | << "\n" |
| 788 | << "Details: " << vpx_codec_error(encoder_) << "\n" |
| 789 | << vpx_codec_error_detail(encoder_); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 790 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 791 | } |
| 792 | timestamp_ += duration; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 793 | |
Sergey Silkin | bc0f0d3 | 2018-04-24 21:29:14 +0200 | [diff] [blame] | 794 | const bool end_of_picture = true; |
| 795 | DeliverBufferedFrame(end_of_picture); |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 796 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 797 | return WEBRTC_VIDEO_CODEC_OK; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 798 | } |
| 799 | |
| 800 | void VP9EncoderImpl::PopulateCodecSpecific(CodecSpecificInfo* codec_specific, |
Niels Möller | d3b8c63 | 2018-08-27 15:33:42 +0200 | [diff] [blame] | 801 | absl::optional<int>* spatial_idx, |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 802 | const vpx_codec_cx_pkt& pkt, |
Sergey Silkin | 07f80cc | 2018-04-09 13:11:59 +0200 | [diff] [blame] | 803 | uint32_t timestamp, |
| 804 | bool first_frame_in_picture) { |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 805 | RTC_CHECK(codec_specific != nullptr); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 806 | codec_specific->codecType = kVideoCodecVP9; |
perkj | 275afc5 | 2016-09-01 00:21:16 -0700 | [diff] [blame] | 807 | codec_specific->codec_name = ImplementationName(); |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 808 | CodecSpecificInfoVP9* vp9_info = &(codec_specific->codecSpecific.VP9); |
Sergey Silkin | 07f80cc | 2018-04-09 13:11:59 +0200 | [diff] [blame] | 809 | |
| 810 | vp9_info->first_frame_in_picture = first_frame_in_picture; |
Sergey Silkin | 738b7e9 | 2018-08-23 16:37:27 +0200 | [diff] [blame] | 811 | vp9_info->flexible_mode = is_flexible_mode_; |
hta | 257dc39 | 2016-10-25 09:05:06 -0700 | [diff] [blame] | 812 | vp9_info->ss_data_available = |
Sergey Silkin | 738b7e9 | 2018-08-23 16:37:27 +0200 | [diff] [blame] | 813 | (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 814 | |
| 815 | vpx_svc_layer_id_t layer_id = {0}; |
| 816 | vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id); |
| 817 | |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 818 | RTC_CHECK_GT(num_temporal_layers_, 0); |
“Michael | 23c5a99 | 2018-06-21 11:07:21 -0500 | [diff] [blame] | 819 | RTC_CHECK_GT(num_active_spatial_layers_, 0); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 820 | if (num_temporal_layers_ == 1) { |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 821 | RTC_CHECK_EQ(layer_id.temporal_layer_id, 0); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 822 | vp9_info->temporal_idx = kNoTemporalIdx; |
| 823 | } else { |
| 824 | vp9_info->temporal_idx = layer_id.temporal_layer_id; |
| 825 | } |
“Michael | 23c5a99 | 2018-06-21 11:07:21 -0500 | [diff] [blame] | 826 | if (num_active_spatial_layers_ == 1) { |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 827 | RTC_CHECK_EQ(layer_id.spatial_layer_id, 0); |
Niels Möller | d3b8c63 | 2018-08-27 15:33:42 +0200 | [diff] [blame] | 828 | *spatial_idx = absl::nullopt; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 829 | } else { |
Niels Möller | d3b8c63 | 2018-08-27 15:33:42 +0200 | [diff] [blame] | 830 | *spatial_idx = layer_id.spatial_layer_id; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 831 | } |
| 832 | if (layer_id.spatial_layer_id != 0) { |
| 833 | vp9_info->ss_data_available = false; |
| 834 | } |
| 835 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 836 | // TODO(asapersson): this info has to be obtained from the encoder. |
asapersson | cb50c96 | 2015-11-18 01:58:55 -0800 | [diff] [blame] | 837 | vp9_info->temporal_up_switch = false; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 838 | |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 839 | if (pkt.data.frame.flags & VPX_FRAME_IS_KEY) { |
| 840 | pics_since_key_ = 0; |
| 841 | } else if (first_frame_in_picture) { |
| 842 | ++pics_since_key_; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 843 | } |
| 844 | |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 845 | const bool is_key_pic = (pics_since_key_ == 0); |
| 846 | const bool is_inter_layer_pred_allowed = |
| 847 | (inter_layer_pred_ == InterLayerPredMode::kOn || |
| 848 | (inter_layer_pred_ == InterLayerPredMode::kOnKeyPic && is_key_pic)); |
| 849 | |
| 850 | // Always set inter_layer_predicted to true on high layer frame if inter-layer |
| 851 | // prediction (ILP) is allowed even if encoder didn't actually use it. |
| 852 | // Setting inter_layer_predicted to false would allow receiver to decode high |
| 853 | // layer frame without decoding low layer frame. If that would happen (e.g. |
| 854 | // if low layer frame is lost) then receiver won't be able to decode next high |
| 855 | // layer frame which uses ILP. |
| 856 | vp9_info->inter_layer_predicted = |
| 857 | first_frame_in_picture ? false : is_inter_layer_pred_allowed; |
| 858 | |
| 859 | const bool is_last_layer = |
“Michael | 23c5a99 | 2018-06-21 11:07:21 -0500 | [diff] [blame] | 860 | (layer_id.spatial_layer_id + 1 == num_active_spatial_layers_); |
Sergey Silkin | 6a8f30e | 2018-04-26 11:03:49 +0200 | [diff] [blame] | 861 | vp9_info->non_ref_for_inter_layer_pred = |
| 862 | is_last_layer ? true : !is_inter_layer_pred_allowed; |
asapersson | 00ac85e | 2015-11-11 05:30:48 -0800 | [diff] [blame] | 863 | |
ivica | 7f6a6fc | 2015-09-08 02:40:29 -0700 | [diff] [blame] | 864 | // Always populate this, so that the packetizer can properly set the marker |
| 865 | // bit. |
“Michael | 23c5a99 | 2018-06-21 11:07:21 -0500 | [diff] [blame] | 866 | vp9_info->num_spatial_layers = num_active_spatial_layers_; |
philipel | cfc319b | 2015-11-10 07:17:23 -0800 | [diff] [blame] | 867 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 868 | vp9_info->num_ref_pics = 0; |
| 869 | if (vp9_info->flexible_mode) { |
| 870 | vp9_info->gof_idx = kNoGofIdx; |
| 871 | FillReferenceIndices(pkt, pics_since_key_, vp9_info->inter_layer_predicted, |
| 872 | vp9_info); |
| 873 | } else { |
| 874 | vp9_info->gof_idx = |
| 875 | static_cast<uint8_t>(pics_since_key_ % gof_.num_frames_in_gof); |
| 876 | vp9_info->temporal_up_switch = gof_.temporal_up_switch[vp9_info->gof_idx]; |
| 877 | vp9_info->num_ref_pics = gof_.num_ref_pics[vp9_info->gof_idx]; |
| 878 | } |
| 879 | |
| 880 | vp9_info->inter_pic_predicted = (!is_key_pic && vp9_info->num_ref_pics > 0); |
| 881 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 882 | if (vp9_info->ss_data_available) { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 883 | vp9_info->spatial_layer_resolution_present = true; |
“Michael | 23c5a99 | 2018-06-21 11:07:21 -0500 | [diff] [blame] | 884 | for (size_t i = 0; i < num_active_spatial_layers_; ++i) { |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 885 | vp9_info->width[i] = codec_.width * svc_params_.scaling_factor_num[i] / |
johannkoenig | 8225c40 | 2017-01-26 13:23:44 -0800 | [diff] [blame] | 886 | svc_params_.scaling_factor_den[i]; |
Yves Gerey | 665174f | 2018-06-19 15:03:05 +0200 | [diff] [blame] | 887 | vp9_info->height[i] = codec_.height * svc_params_.scaling_factor_num[i] / |
johannkoenig | 8225c40 | 2017-01-26 13:23:44 -0800 | [diff] [blame] | 888 | svc_params_.scaling_factor_den[i]; |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 889 | } |
Sergey Silkin | 738b7e9 | 2018-08-23 16:37:27 +0200 | [diff] [blame] | 890 | if (vp9_info->flexible_mode) { |
| 891 | vp9_info->gof.num_frames_in_gof = 0; |
| 892 | } else { |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 893 | vp9_info->gof.CopyGofInfoVP9(gof_); |
| 894 | } |
| 895 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 896 | } |
| 897 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 898 | void VP9EncoderImpl::FillReferenceIndices(const vpx_codec_cx_pkt& pkt, |
| 899 | const size_t pic_num, |
| 900 | const bool inter_layer_predicted, |
| 901 | CodecSpecificInfoVP9* vp9_info) { |
| 902 | vpx_svc_layer_id_t layer_id = {0}; |
| 903 | vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id); |
| 904 | |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 905 | const bool is_key_frame = |
| 906 | (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false; |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 907 | |
| 908 | std::vector<RefFrameBuffer> ref_buf_list; |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 909 | |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 910 | if (is_svc_) { |
| 911 | vpx_svc_ref_frame_config_t enc_layer_conf = {{0}}; |
| 912 | vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf); |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 913 | |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 914 | if (enc_layer_conf.reference_last[layer_id.spatial_layer_id]) { |
| 915 | const size_t fb_idx = |
| 916 | enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id]; |
| 917 | RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end()); |
Sergey Silkin | 8227659 | 2018-08-27 14:54:20 +0200 | [diff] [blame] | 918 | if (std::find(ref_buf_list.begin(), ref_buf_list.end(), |
| 919 | ref_buf_.at(fb_idx)) == ref_buf_list.end()) { |
| 920 | ref_buf_list.push_back(ref_buf_.at(fb_idx)); |
| 921 | } |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 922 | } |
| 923 | |
| 924 | if (enc_layer_conf.reference_alt_ref[layer_id.spatial_layer_id]) { |
| 925 | const size_t fb_idx = |
| 926 | enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id]; |
| 927 | RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end()); |
Sergey Silkin | 8227659 | 2018-08-27 14:54:20 +0200 | [diff] [blame] | 928 | if (std::find(ref_buf_list.begin(), ref_buf_list.end(), |
| 929 | ref_buf_.at(fb_idx)) == ref_buf_list.end()) { |
| 930 | ref_buf_list.push_back(ref_buf_.at(fb_idx)); |
| 931 | } |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 932 | } |
| 933 | |
| 934 | if (enc_layer_conf.reference_golden[layer_id.spatial_layer_id]) { |
| 935 | const size_t fb_idx = |
| 936 | enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id]; |
| 937 | RTC_DCHECK(ref_buf_.find(fb_idx) != ref_buf_.end()); |
Sergey Silkin | 8227659 | 2018-08-27 14:54:20 +0200 | [diff] [blame] | 938 | if (std::find(ref_buf_list.begin(), ref_buf_list.end(), |
| 939 | ref_buf_.at(fb_idx)) == ref_buf_list.end()) { |
| 940 | ref_buf_list.push_back(ref_buf_.at(fb_idx)); |
| 941 | } |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 942 | } |
| 943 | } else if (!is_key_frame) { |
| 944 | RTC_DCHECK_EQ(num_spatial_layers_, 1); |
| 945 | RTC_DCHECK_EQ(num_temporal_layers_, 1); |
| 946 | // In non-SVC mode encoder doesn't provide reference list. Assume each frame |
| 947 | // refers previous one, which is stored in buffer 0. |
| 948 | ref_buf_list.push_back(ref_buf_.at(0)); |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 949 | } |
| 950 | |
| 951 | size_t max_ref_temporal_layer_id = 0; |
| 952 | |
| 953 | vp9_info->num_ref_pics = 0; |
| 954 | for (const RefFrameBuffer& ref_buf : ref_buf_list) { |
| 955 | RTC_DCHECK_LE(ref_buf.pic_num, pic_num); |
| 956 | if (ref_buf.pic_num < pic_num) { |
| 957 | if (inter_layer_pred_ != InterLayerPredMode::kOn) { |
| 958 | // RTP spec limits temporal prediction to the same spatial layer. |
| 959 | // It is safe to ignore this requirement if inter-layer prediction is |
| 960 | // enabled for all frames when all base frames are relayed to receiver. |
| 961 | RTC_DCHECK_EQ(ref_buf.spatial_layer_id, layer_id.spatial_layer_id); |
| 962 | } |
| 963 | RTC_DCHECK_LE(ref_buf.temporal_layer_id, layer_id.temporal_layer_id); |
| 964 | |
| 965 | const size_t p_diff = pic_num - ref_buf.pic_num; |
| 966 | RTC_DCHECK_LE(p_diff, 127UL); |
| 967 | |
| 968 | vp9_info->p_diff[vp9_info->num_ref_pics] = static_cast<uint8_t>(p_diff); |
| 969 | ++vp9_info->num_ref_pics; |
| 970 | |
| 971 | max_ref_temporal_layer_id = |
| 972 | std::max(max_ref_temporal_layer_id, ref_buf.temporal_layer_id); |
| 973 | } else { |
| 974 | RTC_DCHECK(inter_layer_predicted); |
| 975 | // RTP spec only allows to use previous spatial layer for inter-layer |
| 976 | // prediction. |
| 977 | RTC_DCHECK_EQ(ref_buf.spatial_layer_id + 1, layer_id.spatial_layer_id); |
| 978 | } |
| 979 | } |
| 980 | |
| 981 | vp9_info->temporal_up_switch = |
| 982 | (max_ref_temporal_layer_id < |
| 983 | static_cast<size_t>(layer_id.temporal_layer_id)); |
| 984 | } |
| 985 | |
| 986 | void VP9EncoderImpl::UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt, |
| 987 | const size_t pic_num) { |
| 988 | vpx_svc_layer_id_t layer_id = {0}; |
| 989 | vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id); |
| 990 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 991 | const bool is_key_frame = |
| 992 | (pkt.data.frame.flags & VPX_FRAME_IS_KEY) ? true : false; |
| 993 | |
| 994 | RefFrameBuffer frame_buf(pic_num, layer_id.spatial_layer_id, |
| 995 | layer_id.temporal_layer_id); |
| 996 | |
| 997 | if (is_key_frame && layer_id.spatial_layer_id == 0) { |
| 998 | // Key frame updates all ref buffers. |
| 999 | for (size_t i = 0; i < kNumVp9Buffers; ++i) { |
| 1000 | ref_buf_[i] = frame_buf; |
| 1001 | } |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 1002 | } else if (is_svc_) { |
| 1003 | vpx_svc_ref_frame_config_t enc_layer_conf = {{0}}; |
| 1004 | vpx_codec_control(encoder_, VP9E_GET_SVC_REF_FRAME_CONFIG, &enc_layer_conf); |
| 1005 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 1006 | if (enc_layer_conf.update_last[layer_id.spatial_layer_id]) { |
| 1007 | ref_buf_[enc_layer_conf.lst_fb_idx[layer_id.spatial_layer_id]] = |
| 1008 | frame_buf; |
| 1009 | } |
| 1010 | |
| 1011 | if (enc_layer_conf.update_alt_ref[layer_id.spatial_layer_id]) { |
| 1012 | ref_buf_[enc_layer_conf.alt_fb_idx[layer_id.spatial_layer_id]] = |
| 1013 | frame_buf; |
| 1014 | } |
| 1015 | |
| 1016 | if (enc_layer_conf.update_golden[layer_id.spatial_layer_id]) { |
| 1017 | ref_buf_[enc_layer_conf.gld_fb_idx[layer_id.spatial_layer_id]] = |
| 1018 | frame_buf; |
| 1019 | } |
Sergey Silkin | 76be295 | 2018-08-21 12:30:33 +0200 | [diff] [blame] | 1020 | } else { |
| 1021 | RTC_DCHECK_EQ(num_spatial_layers_, 1); |
| 1022 | RTC_DCHECK_EQ(num_temporal_layers_, 1); |
| 1023 | // In non-svc mode encoder doesn't provide reference list. Assume each frame |
| 1024 | // is reference and stored in buffer 0. |
| 1025 | ref_buf_[0] = frame_buf; |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 1026 | } |
| 1027 | } |
| 1028 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 1029 | int VP9EncoderImpl::GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt) { |
asapersson | 86956de | 2016-01-26 01:05:20 -0800 | [diff] [blame] | 1030 | RTC_DCHECK_EQ(pkt->kind, VPX_CODEC_CX_FRAME_PKT); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 1031 | |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1032 | if (pkt->data.frame.sz == 0) { |
| 1033 | // Ignore dropped frame. |
| 1034 | return WEBRTC_VIDEO_CODEC_OK; |
| 1035 | } |
| 1036 | |
Sergey Silkin | 07f80cc | 2018-04-09 13:11:59 +0200 | [diff] [blame] | 1037 | vpx_svc_layer_id_t layer_id = {0}; |
| 1038 | vpx_codec_control(encoder_, VP9E_GET_SVC_LAYER_ID, &layer_id); |
| 1039 | |
| 1040 | const bool first_frame_in_picture = encoded_image_._length == 0; |
| 1041 | // Ensure we don't buffer layers of previous picture (superframe). |
| 1042 | RTC_DCHECK(first_frame_in_picture || layer_id.spatial_layer_id > 0); |
| 1043 | |
Sergey Silkin | bc0f0d3 | 2018-04-24 21:29:14 +0200 | [diff] [blame] | 1044 | const bool end_of_picture = false; |
| 1045 | DeliverBufferedFrame(end_of_picture); |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1046 | |
asapersson | d9f641e | 2016-01-21 01:11:35 -0800 | [diff] [blame] | 1047 | if (pkt->data.frame.sz > encoded_image_._size) { |
| 1048 | delete[] encoded_image_._buffer; |
| 1049 | encoded_image_._size = pkt->data.frame.sz; |
| 1050 | encoded_image_._buffer = new uint8_t[encoded_image_._size]; |
| 1051 | } |
asapersson | 86956de | 2016-01-26 01:05:20 -0800 | [diff] [blame] | 1052 | memcpy(encoded_image_._buffer, pkt->data.frame.buf, pkt->data.frame.sz); |
| 1053 | encoded_image_._length = pkt->data.frame.sz; |
asapersson | d9f641e | 2016-01-21 01:11:35 -0800 | [diff] [blame] | 1054 | |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 1055 | const bool is_key_frame = |
| 1056 | (pkt->data.frame.flags & VPX_FRAME_IS_KEY) ? true : false; |
| 1057 | // Ensure encoder issued key frame on request. |
| 1058 | RTC_DCHECK(is_key_frame || !force_key_frame_); |
| 1059 | |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 1060 | // Check if encoded frame is a key frame. |
asapersson | 86956de | 2016-01-26 01:05:20 -0800 | [diff] [blame] | 1061 | encoded_image_._frameType = kVideoFrameDelta; |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 1062 | if (is_key_frame) { |
Peter Boström | 49e196a | 2015-10-23 15:58:18 +0200 | [diff] [blame] | 1063 | encoded_image_._frameType = kVideoFrameKey; |
Sergey Silkin | bd0954e | 2018-05-03 14:14:09 +0200 | [diff] [blame] | 1064 | force_key_frame_ = false; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1065 | } |
asapersson | 86956de | 2016-01-26 01:05:20 -0800 | [diff] [blame] | 1066 | RTC_DCHECK_LE(encoded_image_._length, encoded_image_._size); |
| 1067 | |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1068 | memset(&codec_specific_, 0, sizeof(codec_specific_)); |
Niels Möller | d3b8c63 | 2018-08-27 15:33:42 +0200 | [diff] [blame] | 1069 | absl::optional<int> spatial_index; |
| 1070 | PopulateCodecSpecific(&codec_specific_, &spatial_index, *pkt, |
| 1071 | input_image_->timestamp(), first_frame_in_picture); |
| 1072 | encoded_image_.SetSpatialIndex(spatial_index); |
asapersson | a9455ab | 2015-07-31 06:10:09 -0700 | [diff] [blame] | 1073 | |
Sergey Silkin | 4e6cd5e | 2018-05-28 12:26:36 +0200 | [diff] [blame] | 1074 | if (is_flexible_mode_) { |
| 1075 | UpdateReferenceBuffers(*pkt, pics_since_key_); |
| 1076 | } |
| 1077 | |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1078 | TRACE_COUNTER1("webrtc", "EncodedFrameSize", encoded_image_._length); |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 1079 | encoded_image_.SetTimestamp(input_image_->timestamp()); |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1080 | encoded_image_.capture_time_ms_ = input_image_->render_time_ms(); |
| 1081 | encoded_image_.rotation_ = input_image_->rotation(); |
Niels Möller | e3cf3d0 | 2018-06-13 11:52:16 +0200 | [diff] [blame] | 1082 | encoded_image_.content_type_ = (codec_.mode == VideoCodecMode::kScreensharing) |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1083 | ? VideoContentType::SCREENSHARE |
| 1084 | : VideoContentType::UNSPECIFIED; |
| 1085 | encoded_image_._encodedHeight = |
| 1086 | pkt->data.frame.height[layer_id.spatial_layer_id]; |
| 1087 | encoded_image_._encodedWidth = |
| 1088 | pkt->data.frame.width[layer_id.spatial_layer_id]; |
Ilya Nikolaevskiy | b6c462d | 2018-06-05 15:21:32 +0200 | [diff] [blame] | 1089 | encoded_image_.timing_.flags = VideoSendTiming::kInvalid; |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1090 | int qp = -1; |
| 1091 | vpx_codec_control(encoder_, VP8E_GET_LAST_QUANTIZER, &qp); |
| 1092 | encoded_image_.qp_ = qp; |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 1093 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1094 | return WEBRTC_VIDEO_CODEC_OK; |
| 1095 | } |
| 1096 | |
Sergey Silkin | bc0f0d3 | 2018-04-24 21:29:14 +0200 | [diff] [blame] | 1097 | void VP9EncoderImpl::DeliverBufferedFrame(bool end_of_picture) { |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1098 | if (encoded_image_._length > 0) { |
Sergey Silkin | bc0f0d3 | 2018-04-24 21:29:14 +0200 | [diff] [blame] | 1099 | codec_specific_.codecSpecific.VP9.end_of_picture = end_of_picture; |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1100 | |
| 1101 | // No data partitioning in VP9, so 1 partition only. |
| 1102 | int part_idx = 0; |
| 1103 | RTPFragmentationHeader frag_info; |
| 1104 | frag_info.VerifyAndAllocateFragmentationHeader(1); |
| 1105 | frag_info.fragmentationOffset[part_idx] = 0; |
| 1106 | frag_info.fragmentationLength[part_idx] = encoded_image_._length; |
| 1107 | frag_info.fragmentationPlType[part_idx] = 0; |
| 1108 | frag_info.fragmentationTimeDiff[part_idx] = 0; |
| 1109 | |
| 1110 | encoded_complete_callback_->OnEncodedImage(encoded_image_, &codec_specific_, |
| 1111 | &frag_info); |
| 1112 | encoded_image_._length = 0; |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 1113 | |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 1114 | if (codec_.mode == VideoCodecMode::kScreensharing) { |
| 1115 | const uint8_t spatial_idx = encoded_image_.SpatialIndex().value_or(0); |
| 1116 | const uint32_t frame_timestamp_ms = |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 1117 | 1000 * encoded_image_.Timestamp() / kVideoPayloadTypeFrequency; |
Sergey Silkin | 96f2c97 | 2018-09-05 21:07:17 +0200 | [diff] [blame] | 1118 | framerate_controller_[spatial_idx].AddFrame(frame_timestamp_ms); |
Sergey Silkin | d902d58 | 2018-05-18 17:31:19 +0200 | [diff] [blame] | 1119 | } |
Sergey Silkin | 2a1f183 | 2018-04-04 11:45:41 +0200 | [diff] [blame] | 1120 | } |
| 1121 | } |
| 1122 | |
pkasting@chromium.org | 16825b1 | 2015-01-12 21:51:21 +0000 | [diff] [blame] | 1123 | int VP9EncoderImpl::SetChannelParameters(uint32_t packet_loss, int64_t rtt) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1124 | return WEBRTC_VIDEO_CODEC_OK; |
| 1125 | } |
| 1126 | |
| 1127 | int VP9EncoderImpl::RegisterEncodeCompleteCallback( |
| 1128 | EncodedImageCallback* callback) { |
| 1129 | encoded_complete_callback_ = callback; |
| 1130 | return WEBRTC_VIDEO_CODEC_OK; |
| 1131 | } |
| 1132 | |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 1133 | const char* VP9EncoderImpl::ImplementationName() const { |
| 1134 | return "libvpx"; |
| 1135 | } |
| 1136 | |
Magnus Jedvert | 46a2765 | 2017-11-13 14:10:02 +0100 | [diff] [blame] | 1137 | std::unique_ptr<VP9Decoder> VP9Decoder::Create() { |
Karl Wiberg | 918f50c | 2018-07-05 11:40:33 +0200 | [diff] [blame] | 1138 | return absl::make_unique<VP9DecoderImpl>(); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1139 | } |
| 1140 | |
| 1141 | VP9DecoderImpl::VP9DecoderImpl() |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1142 | : decode_complete_callback_(nullptr), |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1143 | inited_(false), |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1144 | decoder_(nullptr), |
philipel | 9d7d75b | 2018-04-04 13:03:01 +0200 | [diff] [blame] | 1145 | key_frame_required_(true) {} |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1146 | |
| 1147 | VP9DecoderImpl::~VP9DecoderImpl() { |
| 1148 | inited_ = true; // in order to do the actual release |
| 1149 | Release(); |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1150 | int num_buffers_in_use = frame_buffer_pool_.GetNumBuffersInUse(); |
| 1151 | if (num_buffers_in_use > 0) { |
| 1152 | // The frame buffers are reference counted and frames are exposed after |
| 1153 | // decoding. There may be valid usage cases where previous frames are still |
| 1154 | // referenced after ~VP9DecoderImpl that is not a leak. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 1155 | RTC_LOG(LS_INFO) << num_buffers_in_use << " Vp9FrameBuffers are still " |
| 1156 | << "referenced during ~VP9DecoderImpl."; |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1157 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1158 | } |
| 1159 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1160 | int VP9DecoderImpl::InitDecode(const VideoCodec* inst, int number_of_cores) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1161 | int ret_val = Release(); |
| 1162 | if (ret_val < 0) { |
| 1163 | return ret_val; |
| 1164 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1165 | if (decoder_ == nullptr) { |
pbos@webrtc.org | e728ee0 | 2014-12-17 13:43:55 +0000 | [diff] [blame] | 1166 | decoder_ = new vpx_codec_ctx_t; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1167 | } |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 1168 | vpx_codec_dec_cfg_t cfg; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1169 | // Setting number of threads to a constant value (1) |
| 1170 | cfg.threads = 1; |
| 1171 | cfg.h = cfg.w = 0; // set after decode |
| 1172 | vpx_codec_flags_t flags = 0; |
| 1173 | if (vpx_codec_dec_init(decoder_, vpx_codec_vp9_dx(), &cfg, flags)) { |
| 1174 | return WEBRTC_VIDEO_CODEC_MEMORY; |
| 1175 | } |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1176 | |
| 1177 | if (!frame_buffer_pool_.InitializeVpxUsePool(decoder_)) { |
| 1178 | return WEBRTC_VIDEO_CODEC_MEMORY; |
| 1179 | } |
| 1180 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1181 | inited_ = true; |
| 1182 | // Always start with a complete key frame. |
| 1183 | key_frame_required_ = true; |
| 1184 | return WEBRTC_VIDEO_CODEC_OK; |
| 1185 | } |
| 1186 | |
| 1187 | int VP9DecoderImpl::Decode(const EncodedImage& input_image, |
| 1188 | bool missing_frames, |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1189 | const CodecSpecificInfo* codec_specific_info, |
| 1190 | int64_t /*render_time_ms*/) { |
| 1191 | if (!inited_) { |
| 1192 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 1193 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1194 | if (decode_complete_callback_ == nullptr) { |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1195 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 1196 | } |
| 1197 | // Always start with a complete key frame. |
| 1198 | if (key_frame_required_) { |
Peter Boström | 49e196a | 2015-10-23 15:58:18 +0200 | [diff] [blame] | 1199 | if (input_image._frameType != kVideoFrameKey) |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1200 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 1201 | // We have a key frame - is it complete? |
| 1202 | if (input_image._completeFrame) { |
| 1203 | key_frame_required_ = false; |
| 1204 | } else { |
| 1205 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 1206 | } |
| 1207 | } |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1208 | vpx_codec_iter_t iter = nullptr; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1209 | vpx_image_t* img; |
| 1210 | uint8_t* buffer = input_image._buffer; |
| 1211 | if (input_image._length == 0) { |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1212 | buffer = nullptr; // Triggers full frame concealment. |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1213 | } |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1214 | // During decode libvpx may get and release buffers from |frame_buffer_pool_|. |
| 1215 | // In practice libvpx keeps a few (~3-4) buffers alive at a time. |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 1216 | if (vpx_codec_decode(decoder_, buffer, |
| 1217 | static_cast<unsigned int>(input_image._length), 0, |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1218 | VPX_DL_REALTIME)) { |
| 1219 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 1220 | } |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1221 | // |img->fb_priv| contains the image data, a reference counted Vp9FrameBuffer. |
| 1222 | // It may be released by libvpx during future vpx_codec_decode or |
| 1223 | // vpx_codec_destroy calls. |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1224 | img = vpx_codec_get_frame(decoder_, &iter); |
sakal | 7adadb1 | 2017-02-23 02:54:57 -0800 | [diff] [blame] | 1225 | int qp; |
| 1226 | vpx_codec_err_t vpx_ret = |
| 1227 | vpx_codec_control(decoder_, VPXD_GET_LAST_QUANTIZER, &qp); |
| 1228 | RTC_DCHECK_EQ(vpx_ret, VPX_CODEC_OK); |
| 1229 | int ret = |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 1230 | ReturnFrame(img, input_image.Timestamp(), input_image.ntp_time_ms_, qp); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1231 | if (ret != 0) { |
| 1232 | return ret; |
| 1233 | } |
| 1234 | return WEBRTC_VIDEO_CODEC_OK; |
| 1235 | } |
| 1236 | |
asapersson | 1490f7a | 2016-09-23 02:09:46 -0700 | [diff] [blame] | 1237 | int VP9DecoderImpl::ReturnFrame(const vpx_image_t* img, |
| 1238 | uint32_t timestamp, |
sakal | 7adadb1 | 2017-02-23 02:54:57 -0800 | [diff] [blame] | 1239 | int64_t ntp_time_ms, |
| 1240 | int qp) { |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1241 | if (img == nullptr) { |
| 1242 | // Decoder OK and nullptr image => No show frame. |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1243 | return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| 1244 | } |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1245 | |
| 1246 | // This buffer contains all of |img|'s image data, a reference counted |
perkj | 14f4144 | 2015-11-30 22:15:45 -0800 | [diff] [blame] | 1247 | // Vp9FrameBuffer. (libvpx is done with the buffers after a few |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1248 | // vpx_codec_decode calls or vpx_codec_destroy). |
| 1249 | Vp9FrameBufferPool::Vp9FrameBuffer* img_buffer = |
| 1250 | static_cast<Vp9FrameBufferPool::Vp9FrameBuffer*>(img->fb_priv); |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 1251 | |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 1252 | // The buffer can be used directly by the VideoFrame (without copy) by |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 1253 | // using a Wrapped*Buffer. |
| 1254 | rtc::scoped_refptr<VideoFrameBuffer> img_wrapped_buffer; |
| 1255 | switch (img->bit_depth) { |
| 1256 | case 8: |
| 1257 | img_wrapped_buffer = WrapI420Buffer( |
philipel | cce46fc | 2015-12-21 03:04:49 -0800 | [diff] [blame] | 1258 | img->d_w, img->d_h, img->planes[VPX_PLANE_Y], |
| 1259 | img->stride[VPX_PLANE_Y], img->planes[VPX_PLANE_U], |
| 1260 | img->stride[VPX_PLANE_U], img->planes[VPX_PLANE_V], |
| 1261 | img->stride[VPX_PLANE_V], |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1262 | // WrappedI420Buffer's mechanism for allowing the release of its frame |
| 1263 | // buffer is through a callback function. This is where we should |
| 1264 | // release |img_buffer|. |
Emircan Uysaler | 0823eec | 2018-07-13 17:10:00 -0700 | [diff] [blame] | 1265 | rtc::KeepRefUntilDone(img_buffer)); |
| 1266 | break; |
| 1267 | case 10: |
| 1268 | img_wrapped_buffer = WrapI010Buffer( |
| 1269 | img->d_w, img->d_h, |
| 1270 | reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_Y]), |
| 1271 | img->stride[VPX_PLANE_Y] / 2, |
| 1272 | reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_U]), |
| 1273 | img->stride[VPX_PLANE_U] / 2, |
| 1274 | reinterpret_cast<const uint16_t*>(img->planes[VPX_PLANE_V]), |
| 1275 | img->stride[VPX_PLANE_V] / 2, rtc::KeepRefUntilDone(img_buffer)); |
| 1276 | break; |
| 1277 | default: |
| 1278 | RTC_NOTREACHED(); |
| 1279 | return WEBRTC_VIDEO_CODEC_NO_OUTPUT; |
| 1280 | } |
nisse | ca6d5d1 | 2016-06-17 05:03:04 -0700 | [diff] [blame] | 1281 | |
Emircan Uysaler | 715cc23 | 2018-07-23 17:50:43 -0700 | [diff] [blame] | 1282 | VideoFrame decoded_image = VideoFrame::Builder() |
| 1283 | .set_video_frame_buffer(img_wrapped_buffer) |
| 1284 | .set_timestamp_ms(0) |
| 1285 | .set_timestamp_rtp(timestamp) |
| 1286 | .set_ntp_time_ms(ntp_time_ms) |
| 1287 | .set_rotation(webrtc::kVideoRotation_0) |
| 1288 | .set_color_space(ExtractVP9ColorSpace( |
| 1289 | img->cs, img->range, img->bit_depth)) |
| 1290 | .build(); |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 1291 | decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1292 | return WEBRTC_VIDEO_CODEC_OK; |
| 1293 | } |
| 1294 | |
| 1295 | int VP9DecoderImpl::RegisterDecodeCompleteCallback( |
| 1296 | DecodedImageCallback* callback) { |
| 1297 | decode_complete_callback_ = callback; |
| 1298 | return WEBRTC_VIDEO_CODEC_OK; |
| 1299 | } |
| 1300 | |
| 1301 | int VP9DecoderImpl::Release() { |
Sergey Silkin | 3e871ea | 2018-03-02 13:11:04 +0100 | [diff] [blame] | 1302 | int ret_val = WEBRTC_VIDEO_CODEC_OK; |
| 1303 | |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1304 | if (decoder_ != nullptr) { |
Sergey Silkin | 9039969 | 2018-03-02 14:44:10 +0100 | [diff] [blame] | 1305 | if (inited_) { |
| 1306 | // When a codec is destroyed libvpx will release any buffers of |
| 1307 | // |frame_buffer_pool_| it is currently using. |
| 1308 | if (vpx_codec_destroy(decoder_)) { |
| 1309 | ret_val = WEBRTC_VIDEO_CODEC_MEMORY; |
| 1310 | } |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1311 | } |
| 1312 | delete decoder_; |
sprang | 3958ed8 | 2017-08-17 08:12:10 -0700 | [diff] [blame] | 1313 | decoder_ = nullptr; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1314 | } |
Henrik Boström | 9695d85 | 2015-05-06 10:42:15 +0200 | [diff] [blame] | 1315 | // Releases buffers from the pool. Any buffers not in use are deleted. Buffers |
| 1316 | // still referenced externally are deleted once fully released, not returning |
| 1317 | // to the pool. |
| 1318 | frame_buffer_pool_.ClearPool(); |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1319 | inited_ = false; |
Sergey Silkin | 3e871ea | 2018-03-02 13:11:04 +0100 | [diff] [blame] | 1320 | return ret_val; |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1321 | } |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 1322 | |
| 1323 | const char* VP9DecoderImpl::ImplementationName() const { |
| 1324 | return "libvpx"; |
| 1325 | } |
| 1326 | |
marpan@webrtc.org | 5b88317 | 2014-11-01 06:10:48 +0000 | [diff] [blame] | 1327 | } // namespace webrtc |