blob: 1147d45be8d1a3b77fbca63a0d6bd55f2e36bd3f [file] [log] [blame]
pbos@webrtc.org9115cde2014-12-09 10:36:40 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "media/engine/simulcast_encoder_adapter.h"
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000012
Yves Gerey3e707812018-11-28 16:47:49 +010013#include <stdio.h>
14#include <string.h>
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000015#include <algorithm>
Yves Gerey3e707812018-11-28 16:47:49 +010016#include <cstdint>
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +020017#include <string>
18#include <utility>
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000019
Mirko Bonadeid9708072019-01-25 20:26:48 +010020#include "api/scoped_refptr.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010022#include "api/video/video_codec_constants.h"
Yves Gerey3e707812018-11-28 16:47:49 +010023#include "api/video/video_frame_buffer.h"
24#include "api/video/video_rotation.h"
Magnus Jedvertdf4883d2017-11-17 14:44:55 +010025#include "api/video_codecs/video_encoder_factory.h"
Yves Gerey3e707812018-11-28 16:47:49 +010026#include "modules/video_coding/include/video_error_codes.h"
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +020027#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080028#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "rtc_base/checks.h"
Erik Språng7f24fb92019-02-13 10:49:37 +010030#include "rtc_base/experiments/rate_control_settings.h"
Erik Språng16cb8f52019-04-12 13:59:09 +020031#include "rtc_base/logging.h"
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +020032#include "system_wrappers/include/field_trial.h"
Mirko Bonadei65432062017-12-11 09:32:13 +010033#include "third_party/libyuv/include/libyuv/scale.h"
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000034
35namespace {
36
37const unsigned int kDefaultMinQp = 2;
38const unsigned int kDefaultMaxQp = 56;
39// Max qp for lowest spatial resolution when doing simulcast.
40const unsigned int kLowestResMaxQp = 45;
41
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +020042absl::optional<unsigned int> GetScreenshareBoostedQpValue() {
43 std::string experiment_group =
44 webrtc::field_trial::FindFullName("WebRTC-BoostedScreenshareQp");
45 unsigned int qp;
46 if (sscanf(experiment_group.c_str(), "%u", &qp) != 1)
47 return absl::nullopt;
48 qp = std::min(qp, 63u);
49 qp = std::max(qp, 1u);
50 return qp;
51}
52
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000053uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
54 uint32_t bitrate_sum = 0;
55 for (int i = 0; i < streams; ++i) {
56 bitrate_sum += codec.simulcastStream[i].maxBitrate;
57 }
58 return bitrate_sum;
59}
60
61int NumberOfStreams(const webrtc::VideoCodec& codec) {
62 int streams =
63 codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
64 uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
65 if (simulcast_max_bitrate == 0) {
66 streams = 1;
67 }
68 return streams;
69}
70
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000071int VerifyCodec(const webrtc::VideoCodec* inst) {
brandtr5e171752017-05-23 03:32:16 -070072 if (inst == nullptr) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000073 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
74 }
75 if (inst->maxFramerate < 1) {
76 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
77 }
78 // allow zero to represent an unspecified maxBitRate
79 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
80 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
81 }
82 if (inst->width <= 1 || inst->height <= 1) {
83 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
84 }
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +020085 if (inst->codecType == webrtc::kVideoCodecVP8 &&
86 inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000087 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
88 }
89 return WEBRTC_VIDEO_CODEC_OK;
90}
91
Florent Castelli1b761ca2019-01-21 14:33:02 +010092bool StreamResolutionCompare(const webrtc::SimulcastStream& a,
93 const webrtc::SimulcastStream& b) {
94 return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
95 std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
96}
97
Noah Richards41ee1ea2015-04-15 09:24:26 -070098// An EncodedImageCallback implementation that forwards on calls to a
99// SimulcastEncoderAdapter, but with the stream index it's registered with as
100// the first parameter to Encoded.
101class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback {
102 public:
103 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter,
104 size_t stream_idx)
105 : adapter_(adapter), stream_idx_(stream_idx) {}
106
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700107 EncodedImageCallback::Result OnEncodedImage(
108 const webrtc::EncodedImage& encoded_image,
109 const webrtc::CodecSpecificInfo* codec_specific_info,
110 const webrtc::RTPFragmentationHeader* fragmentation) override {
111 return adapter_->OnEncodedImage(stream_idx_, encoded_image,
112 codec_specific_info, fragmentation);
Noah Richards41ee1ea2015-04-15 09:24:26 -0700113 }
114
115 private:
116 webrtc::SimulcastEncoderAdapter* const adapter_;
117 const size_t stream_idx_;
118};
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000119} // namespace
120
121namespace webrtc {
122
Ilya Nikolaevskiy97b4ee52018-05-28 10:24:22 +0200123SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory,
124 const SdpVideoFormat& format)
Magnus Jedvertdf4883d2017-11-17 14:44:55 +0100125 : inited_(0),
126 factory_(factory),
Ilya Nikolaevskiy97b4ee52018-05-28 10:24:22 +0200127 video_format_(format),
Magnus Jedvertdf4883d2017-11-17 14:44:55 +0100128 encoded_complete_callback_(nullptr),
Erik Språng7f24fb92019-02-13 10:49:37 +0100129 experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
130 boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
131 .Vp8BoostBaseLayerQuality()) {
Rasmus Brandtc334ce92018-02-05 13:03:25 +0100132 RTC_DCHECK(factory_);
Erik Språng75de46a2018-11-07 14:53:32 +0100133 encoder_info_.implementation_name = "SimulcastEncoderAdapter";
Magnus Jedvertdf4883d2017-11-17 14:44:55 +0100134
brandtr5e171752017-05-23 03:32:16 -0700135 // The adapter is typically created on the worker thread, but operated on
136 // the encoder task queue.
137 encoder_queue_.Detach();
138
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000139 memset(&codec_, 0, sizeof(webrtc::VideoCodec));
140}
141
142SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
brandtr5e171752017-05-23 03:32:16 -0700143 RTC_DCHECK(!Initialized());
144 DestroyStoredEncoders();
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000145}
146
147int SimulcastEncoderAdapter::Release() {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200148 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700149
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000150 while (!streaminfos_.empty()) {
magjed3f897582017-08-28 08:05:42 -0700151 std::unique_ptr<VideoEncoder> encoder =
152 std::move(streaminfos_.back().encoder);
brandtr5e171752017-05-23 03:32:16 -0700153 // Even though it seems very unlikely, there are no guarantees that the
Rasmus Brandt9cf9f752017-09-28 13:02:58 +0200154 // encoder will not call back after being Release()'d. Therefore, we first
155 // disable the callbacks here.
brandtr5e171752017-05-23 03:32:16 -0700156 encoder->RegisterEncodeCompleteCallback(nullptr);
Rasmus Brandt9cf9f752017-09-28 13:02:58 +0200157 encoder->Release();
brandtr5e171752017-05-23 03:32:16 -0700158 streaminfos_.pop_back(); // Deletes callback adapter.
magjed3f897582017-08-28 08:05:42 -0700159 stored_encoders_.push(std::move(encoder));
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000160 }
brandtr5e171752017-05-23 03:32:16 -0700161
162 // It's legal to move the encoder to another queue now.
163 encoder_queue_.Detach();
164
165 rtc::AtomicOps::ReleaseStore(&inited_, 0);
166
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000167 return WEBRTC_VIDEO_CODEC_OK;
168}
169
170int SimulcastEncoderAdapter::InitEncode(const VideoCodec* inst,
171 int number_of_cores,
172 size_t max_payload_size) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200173 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700174
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000175 if (number_of_cores < 1) {
176 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
177 }
178
179 int ret = VerifyCodec(inst);
180 if (ret < 0) {
181 return ret;
182 }
183
184 ret = Release();
185 if (ret < 0) {
186 return ret;
187 }
188
189 int number_of_streams = NumberOfStreams(*inst);
brandtr5e171752017-05-23 03:32:16 -0700190 RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
Peter Boströmd53c3892016-03-30 17:03:52 +0200191 const bool doing_simulcast = (number_of_streams > 1);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000192
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000193 codec_ = *inst;
Erik Språng82fad3d2018-03-21 09:57:23 +0100194 SimulcastRateAllocator rate_allocator(codec_);
Erik Språng566124a2018-04-23 12:32:22 +0200195 VideoBitrateAllocation allocation = rate_allocator.GetAllocation(
Erik Språng08127a92016-11-16 16:41:30 +0100196 codec_.startBitrate * 1000, codec_.maxFramerate);
197 std::vector<uint32_t> start_bitrates;
198 for (int i = 0; i < kMaxSimulcastStreams; ++i) {
199 uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
200 start_bitrates.push_back(stream_bitrate);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000201 }
202
Erik Språng75de46a2018-11-07 14:53:32 +0100203 encoder_info_.supports_native_handle = true;
204 encoder_info_.scaling_settings.thresholds = absl::nullopt;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000205 // Create |number_of_streams| of encoder instances and init them.
Florent Castelli1b761ca2019-01-21 14:33:02 +0100206
207 const auto minmax = std::minmax_element(
208 std::begin(codec_.simulcastStream),
209 std::begin(codec_.simulcastStream) + number_of_streams,
210 StreamResolutionCompare);
211 const auto lowest_resolution_stream_index =
212 std::distance(std::begin(codec_.simulcastStream), minmax.first);
213 const auto highest_resolution_stream_index =
214 std::distance(std::begin(codec_.simulcastStream), minmax.second);
215
216 RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
217 RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
218
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000219 for (int i = 0; i < number_of_streams; ++i) {
220 VideoCodec stream_codec;
Erik Språng78ce6192016-09-12 16:04:43 +0200221 uint32_t start_bitrate_kbps = start_bitrates[i];
Erik Språng7d687b12018-09-12 17:04:10 +0200222 const bool send_stream = start_bitrate_kbps > 0;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000223 if (!doing_simulcast) {
224 stream_codec = codec_;
225 stream_codec.numberOfSimulcastStreams = 1;
Erik Språng75de46a2018-11-07 14:53:32 +0100226
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000227 } else {
Erik Språng78ce6192016-09-12 16:04:43 +0200228 // Cap start bitrate to the min bitrate in order to avoid strange codec
Erik Språng7d687b12018-09-12 17:04:10 +0200229 // behavior. Since sending will be false, this should not matter.
Florent Castelli1b761ca2019-01-21 14:33:02 +0100230 StreamResolution stream_resolution =
231 i == highest_resolution_stream_index
232 ? StreamResolution::HIGHEST
233 : i == lowest_resolution_stream_index ? StreamResolution::LOWEST
234 : StreamResolution::OTHER;
235
Erik Språng78ce6192016-09-12 16:04:43 +0200236 start_bitrate_kbps =
237 std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps);
Florent Castelli1b761ca2019-01-21 14:33:02 +0100238 PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
239 &stream_codec);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000240 }
241
Erik Språngcc681cc2018-03-14 17:52:55 +0100242 // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000243 if (stream_codec.qpMax < kDefaultMinQp) {
244 stream_codec.qpMax = kDefaultMaxQp;
245 }
246
brandtr5e171752017-05-23 03:32:16 -0700247 // If an existing encoder instance exists, reuse it.
248 // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here,
249 // when we start storing that state outside the encoder wrappers.
magjed3f897582017-08-28 08:05:42 -0700250 std::unique_ptr<VideoEncoder> encoder;
brandtr5e171752017-05-23 03:32:16 -0700251 if (!stored_encoders_.empty()) {
magjed3f897582017-08-28 08:05:42 -0700252 encoder = std::move(stored_encoders_.top());
brandtr5e171752017-05-23 03:32:16 -0700253 stored_encoders_.pop();
254 } else {
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +0200255 encoder = factory_->CreateVideoEncoder(SdpVideoFormat(
256 codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264"));
brandtr5e171752017-05-23 03:32:16 -0700257 }
258
philipelcce46fc2015-12-21 03:04:49 -0800259 ret = encoder->InitEncode(&stream_codec, number_of_cores, max_payload_size);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000260 if (ret < 0) {
noahrice5ba75a2016-12-12 13:08:27 -0800261 // Explicitly destroy the current encoder; because we haven't registered a
262 // StreamInfo for it yet, Release won't do anything about it.
magjed3f897582017-08-28 08:05:42 -0700263 encoder.reset();
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000264 Release();
265 return ret;
266 }
Erik Språngd3438aa2018-11-08 16:56:43 +0100267
brandtr5e171752017-05-23 03:32:16 -0700268 std::unique_ptr<EncodedImageCallback> callback(
269 new AdapterEncodedImageCallback(this, i));
270 encoder->RegisterEncodeCompleteCallback(callback.get());
magjed3f897582017-08-28 08:05:42 -0700271 streaminfos_.emplace_back(std::move(encoder), std::move(callback),
272 stream_codec.width, stream_codec.height,
Erik Språng7d687b12018-09-12 17:04:10 +0200273 send_stream);
brandtr5e171752017-05-23 03:32:16 -0700274
Erik Språng75de46a2018-11-07 14:53:32 +0100275 if (!doing_simulcast) {
276 // Without simulcast, just pass through the encoder info from the one
277 // active encoder.
278 encoder_info_ = streaminfos_[0].encoder->GetEncoderInfo();
279 } else {
280 const EncoderInfo encoder_impl_info =
281 streaminfos_[i].encoder->GetEncoderInfo();
282
283 if (i == 0) {
284 // Quality scaling not enabled for simulcast.
285 encoder_info_.scaling_settings = VideoEncoder::ScalingSettings::kOff;
286
287 // Encoder name indicates names of all sub-encoders.
288 encoder_info_.implementation_name = "SimulcastEncoderAdapter (";
289 encoder_info_.implementation_name +=
290 encoder_impl_info.implementation_name;
291
292 encoder_info_.supports_native_handle =
293 encoder_impl_info.supports_native_handle;
Erik Språngd3438aa2018-11-08 16:56:43 +0100294 encoder_info_.has_trusted_rate_controller =
295 encoder_impl_info.has_trusted_rate_controller;
Mirta Dvornicic897a9912018-11-30 13:12:21 +0100296 encoder_info_.is_hardware_accelerated =
297 encoder_impl_info.is_hardware_accelerated;
298 encoder_info_.has_internal_source =
299 encoder_impl_info.has_internal_source;
Erik Språng75de46a2018-11-07 14:53:32 +0100300 } else {
301 encoder_info_.implementation_name += ", ";
302 encoder_info_.implementation_name +=
303 encoder_impl_info.implementation_name;
304
305 // Native handle supported only if all encoders supports it.
306 encoder_info_.supports_native_handle &=
307 encoder_impl_info.supports_native_handle;
Erik Språngd3438aa2018-11-08 16:56:43 +0100308
309 // Trusted rate controller only if all encoders have it.
310 encoder_info_.has_trusted_rate_controller &=
311 encoder_impl_info.has_trusted_rate_controller;
Mirta Dvornicic897a9912018-11-30 13:12:21 +0100312
313 // Uses hardware support if any of the encoders uses it.
314 // For example, if we are having issues with down-scaling due to
315 // pipelining delay in HW encoders we need higher encoder usage
316 // thresholds in CPU adaptation.
317 encoder_info_.is_hardware_accelerated |=
318 encoder_impl_info.is_hardware_accelerated;
319
320 // Has internal source only if all encoders have it.
321 encoder_info_.has_internal_source &=
322 encoder_impl_info.has_internal_source;
Erik Språng75de46a2018-11-07 14:53:32 +0100323 }
Erik Språngdbdd8392019-01-17 15:27:50 +0100324 encoder_info_.fps_allocation[i] = encoder_impl_info.fps_allocation[0];
brandtr5e171752017-05-23 03:32:16 -0700325 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000326 }
brandtr5e171752017-05-23 03:32:16 -0700327
Peter Boströmd53c3892016-03-30 17:03:52 +0200328 if (doing_simulcast) {
Erik Språng75de46a2018-11-07 14:53:32 +0100329 encoder_info_.implementation_name += ")";
Peter Boströmd53c3892016-03-30 17:03:52 +0200330 }
brandtr5e171752017-05-23 03:32:16 -0700331
332 // To save memory, don't store encoders that we don't use.
333 DestroyStoredEncoders();
334
335 rtc::AtomicOps::ReleaseStore(&inited_, 1);
336
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000337 return WEBRTC_VIDEO_CODEC_OK;
338}
339
340int SimulcastEncoderAdapter::Encode(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700341 const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100342 const std::vector<VideoFrameType>* frame_types) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200343 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700344
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000345 if (!Initialized()) {
346 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
347 }
brandtr5e171752017-05-23 03:32:16 -0700348 if (encoded_complete_callback_ == nullptr) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000349 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
350 }
351
352 // All active streams should generate a key frame if
353 // a key frame is requested by any stream.
354 bool send_key_frame = false;
355 if (frame_types) {
356 for (size_t i = 0; i < frame_types->size(); ++i) {
Niels Möller8f7ce222019-03-21 15:43:58 +0100357 if (frame_types->at(i) == VideoFrameType::kVideoFrameKey) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000358 send_key_frame = true;
359 break;
360 }
361 }
362 }
363 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
364 if (streaminfos_[stream_idx].key_frame_request &&
365 streaminfos_[stream_idx].send_stream) {
366 send_key_frame = true;
367 break;
368 }
369 }
370
371 int src_width = input_image.width();
372 int src_height = input_image.height();
373 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
Peter Boström5d0379d2015-10-06 14:04:51 +0200374 // Don't encode frames in resolutions that we don't intend to send.
brandtr5e171752017-05-23 03:32:16 -0700375 if (!streaminfos_[stream_idx].send_stream) {
Peter Boström5d0379d2015-10-06 14:04:51 +0200376 continue;
brandtr5e171752017-05-23 03:32:16 -0700377 }
Peter Boström5d0379d2015-10-06 14:04:51 +0200378
Niels Möller87e2d782019-03-07 10:18:23 +0100379 std::vector<VideoFrameType> stream_frame_types;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000380 if (send_key_frame) {
Niels Möller8f7ce222019-03-21 15:43:58 +0100381 stream_frame_types.push_back(VideoFrameType::kVideoFrameKey);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000382 streaminfos_[stream_idx].key_frame_request = false;
383 } else {
Niels Möller8f7ce222019-03-21 15:43:58 +0100384 stream_frame_types.push_back(VideoFrameType::kVideoFrameDelta);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000385 }
386
387 int dst_width = streaminfos_[stream_idx].width;
388 int dst_height = streaminfos_[stream_idx].height;
389 // If scaling isn't required, because the input resolution
390 // matches the destination or the input image is empty (e.g.
391 // a keyframe request for encoders with internal camera
noahricfe3654d2016-07-01 09:05:54 -0700392 // sources) or the source image has a native handle, pass the image on
393 // directly. Otherwise, we'll scale it to match what the encoder expects
394 // (below).
395 // For texture frames, the underlying encoder is expected to be able to
396 // correctly sample/scale the source texture.
397 // TODO(perkj): ensure that works going forward, and figure out how this
398 // affects webrtc:5683.
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000399 if ((dst_width == src_width && dst_height == src_height) ||
Magnus Jedvert72dbe2a2017-06-10 17:03:37 +0000400 input_image.video_frame_buffer()->type() ==
401 VideoFrameBuffer::Type::kNative) {
Niels Möllerc8d2e732019-03-06 12:00:33 +0100402 int ret = streaminfos_[stream_idx].encoder->Encode(input_image,
403 &stream_frame_types);
noahric57779102016-05-25 06:48:46 -0700404 if (ret != WEBRTC_VIDEO_CODEC_OK) {
405 return ret;
406 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000407 } else {
nisse64ec8f82016-09-27 00:17:25 -0700408 rtc::scoped_refptr<I420Buffer> dst_buffer =
Magnus Jedvert72dbe2a2017-06-10 17:03:37 +0000409 I420Buffer::Create(dst_width, dst_height);
410 rtc::scoped_refptr<I420BufferInterface> src_buffer =
411 input_image.video_frame_buffer()->ToI420();
412 libyuv::I420Scale(src_buffer->DataY(), src_buffer->StrideY(),
413 src_buffer->DataU(), src_buffer->StrideU(),
414 src_buffer->DataV(), src_buffer->StrideV(), src_width,
415 src_height, dst_buffer->MutableDataY(),
416 dst_buffer->StrideY(), dst_buffer->MutableDataU(),
417 dst_buffer->StrideU(), dst_buffer->MutableDataV(),
418 dst_buffer->StrideV(), dst_width, dst_height,
nissec9c142f2016-05-17 04:05:47 -0700419 libyuv::kFilterBilinear);
nisse64ec8f82016-09-27 00:17:25 -0700420
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100421 // UpdateRect is not propagated to lower simulcast layers currently.
422 // TODO(ilnik): Consider scaling UpdateRect together with the buffer.
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100423 VideoFrame frame = VideoFrame::Builder()
424 .set_video_frame_buffer(dst_buffer)
425 .set_timestamp_rtp(input_image.timestamp())
426 .set_rotation(webrtc::kVideoRotation_0)
427 .set_timestamp_ms(input_image.render_time_ms())
428 .build();
Niels Möllerc8d2e732019-03-06 12:00:33 +0100429 int ret =
430 streaminfos_[stream_idx].encoder->Encode(frame, &stream_frame_types);
noahric57779102016-05-25 06:48:46 -0700431 if (ret != WEBRTC_VIDEO_CODEC_OK) {
432 return ret;
433 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000434 }
435 }
436
437 return WEBRTC_VIDEO_CODEC_OK;
438}
439
440int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
441 EncodedImageCallback* callback) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200442 RTC_DCHECK_RUN_ON(&encoder_queue_);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000443 encoded_complete_callback_ = callback;
444 return WEBRTC_VIDEO_CODEC_OK;
445}
446
Erik Språng16cb8f52019-04-12 13:59:09 +0200447void SimulcastEncoderAdapter::SetRates(
448 const RateControlParameters& parameters) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200449 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700450
451 if (!Initialized()) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200452 RTC_LOG(LS_WARNING) << "SetRates while not initialized";
453 return;
brandtr5e171752017-05-23 03:32:16 -0700454 }
sprang647bf432016-11-10 06:46:20 -0800455
Erik Språng16cb8f52019-04-12 13:59:09 +0200456 if (parameters.framerate_fps < 1.0) {
457 RTC_LOG(LS_WARNING) << "Invalid framerate: " << parameters.framerate_fps;
458 return;
brandtr5e171752017-05-23 03:32:16 -0700459 }
Erik Språng08127a92016-11-16 16:41:30 +0100460
Erik Språng16cb8f52019-04-12 13:59:09 +0200461 if (codec_.maxBitrate > 0 &&
462 parameters.bitrate.get_sum_kbps() > codec_.maxBitrate) {
463 RTC_LOG(LS_WARNING) << "Total bitrate " << parameters.bitrate.get_sum_kbps()
464 << " exceeds max bitrate: " << codec_.maxBitrate;
465 return;
brandtr5e171752017-05-23 03:32:16 -0700466 }
Erik Språng08127a92016-11-16 16:41:30 +0100467
Erik Språng16cb8f52019-04-12 13:59:09 +0200468 if (parameters.bitrate.get_sum_bps() > 0) {
sprang1369c832016-11-10 08:30:33 -0800469 // Make sure the bitrate fits the configured min bitrates. 0 is a special
470 // value that means paused, though, so leave it alone.
Erik Språng16cb8f52019-04-12 13:59:09 +0200471 if (parameters.bitrate.get_sum_kbps() < codec_.minBitrate) {
472 RTC_LOG(LS_WARNING) << "Total bitrate "
473 << parameters.bitrate.get_sum_kbps()
474 << " is lower than minimum bitrate: "
475 << codec_.minBitrate;
476 return;
brandtr5e171752017-05-23 03:32:16 -0700477 }
Erik Språng08127a92016-11-16 16:41:30 +0100478
sprang1369c832016-11-10 08:30:33 -0800479 if (codec_.numberOfSimulcastStreams > 0 &&
Erik Språng16cb8f52019-04-12 13:59:09 +0200480 parameters.bitrate.get_sum_kbps() <
481 codec_.simulcastStream[0].minBitrate) {
482 RTC_LOG(LS_WARNING) << "Total bitrate "
483 << parameters.bitrate.get_sum_kbps()
484 << " is lower than minimum bitrate of base layer: "
485 << codec_.simulcastStream[0].minBitrate;
486 return;
sprang1369c832016-11-10 08:30:33 -0800487 }
sprang1369c832016-11-10 08:30:33 -0800488 }
Erik Språng08127a92016-11-16 16:41:30 +0100489
Erik Språng16cb8f52019-04-12 13:59:09 +0200490 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000491
sprang1369c832016-11-10 08:30:33 -0800492 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
Erik Språng08127a92016-11-16 16:41:30 +0100493 uint32_t stream_bitrate_kbps =
Erik Språng16cb8f52019-04-12 13:59:09 +0200494 parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
Erik Språng08127a92016-11-16 16:41:30 +0100495
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000496 // Need a key frame if we have not sent this stream before.
Erik Språng78ce6192016-09-12 16:04:43 +0200497 if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000498 streaminfos_[stream_idx].key_frame_request = true;
499 }
Erik Språng78ce6192016-09-12 16:04:43 +0200500 streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000501
Erik Språng08127a92016-11-16 16:41:30 +0100502 // Slice the temporal layers out of the full allocation and pass it on to
503 // the encoder handling the current simulcast stream.
Erik Språng16cb8f52019-04-12 13:59:09 +0200504 RateControlParameters stream_parameters = parameters;
505 stream_parameters.bitrate = VideoBitrateAllocation();
brandtr5e171752017-05-23 03:32:16 -0700506 for (int i = 0; i < kMaxTemporalStreams; ++i) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200507 if (parameters.bitrate.HasBitrate(stream_idx, i)) {
508 stream_parameters.bitrate.SetBitrate(
509 0, i, parameters.bitrate.GetBitrate(stream_idx, i));
erikvarga@webrtc.org01f2ec32017-11-15 14:58:23 +0100510 }
brandtr5e171752017-05-23 03:32:16 -0700511 }
Minyue Li7ddef1a2019-04-11 10:50:19 +0000512
Erik Språng16cb8f52019-04-12 13:59:09 +0200513 // Assign link allocation proportionally to spatial layer allocation.
514 if (parameters.bandwidth_allocation != DataRate::Zero()) {
515 stream_parameters.bandwidth_allocation =
516 DataRate::bps((parameters.bandwidth_allocation.bps() *
517 stream_parameters.bitrate.get_sum_bps()) /
518 parameters.bitrate.get_sum_bps());
519 // Make sure we don't allocate bandwidth lower than target bitrate.
520 if (stream_parameters.bandwidth_allocation.bps() <
521 stream_parameters.bitrate.get_sum_bps()) {
522 stream_parameters.bandwidth_allocation =
523 DataRate::bps(stream_parameters.bitrate.get_sum_bps());
524 }
525 }
526
527 streaminfos_[stream_idx].encoder->SetRates(stream_parameters);
528 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000529}
530
brandtr5e171752017-05-23 03:32:16 -0700531// TODO(brandtr): Add task checker to this member function, when all encoder
532// callbacks are coming in on the encoder queue.
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700533EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
Noah Richards41ee1ea2015-04-15 09:24:26 -0700534 size_t stream_idx,
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000535 const EncodedImage& encodedImage,
536 const CodecSpecificInfo* codecSpecificInfo,
537 const RTPFragmentationHeader* fragmentation) {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200538 EncodedImage stream_image(encodedImage);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000539 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
Niels Möllerd3b8c632018-08-27 15:33:42 +0200540
541 stream_image.SetSpatialIndex(stream_idx);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000542
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700543 return encoded_complete_callback_->OnEncodedImage(
Niels Möllerd3b8c632018-08-27 15:33:42 +0200544 stream_image, &stream_codec_specific, fragmentation);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000545}
546
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000547void SimulcastEncoderAdapter::PopulateStreamCodec(
brandtr5e171752017-05-23 03:32:16 -0700548 const webrtc::VideoCodec& inst,
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000549 int stream_index,
Erik Språng78ce6192016-09-12 16:04:43 +0200550 uint32_t start_bitrate_kbps,
Florent Castelli1b761ca2019-01-21 14:33:02 +0100551 StreamResolution stream_resolution,
Erik Språng78ce6192016-09-12 16:04:43 +0200552 webrtc::VideoCodec* stream_codec) {
brandtr5e171752017-05-23 03:32:16 -0700553 *stream_codec = inst;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000554
555 // Stream specific settings.
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000556 stream_codec->numberOfSimulcastStreams = 0;
brandtr5e171752017-05-23 03:32:16 -0700557 stream_codec->width = inst.simulcastStream[stream_index].width;
558 stream_codec->height = inst.simulcastStream[stream_index].height;
559 stream_codec->maxBitrate = inst.simulcastStream[stream_index].maxBitrate;
560 stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate;
561 stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000562 // Settings that are based on stream/resolution.
Florent Castelli1b761ca2019-01-21 14:33:02 +0100563 if (stream_resolution == StreamResolution::LOWEST) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000564 // Settings for lowest spatial resolutions.
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +0200565 if (inst.mode == VideoCodecMode::kScreensharing) {
566 if (experimental_boosted_screenshare_qp_) {
567 stream_codec->qpMax = *experimental_boosted_screenshare_qp_;
568 }
Erik Språng7f24fb92019-02-13 10:49:37 +0100569 } else if (boost_base_layer_quality_) {
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +0200570 stream_codec->qpMax = kLowestResMaxQp;
571 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000572 }
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +0200573 if (inst.codecType == webrtc::kVideoCodecVP8) {
574 stream_codec->VP8()->numberOfTemporalLayers =
575 inst.simulcastStream[stream_index].numberOfTemporalLayers;
Florent Castelli1b761ca2019-01-21 14:33:02 +0100576 if (stream_resolution != StreamResolution::HIGHEST) {
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +0200577 // For resolutions below CIF, set the codec |complexity| parameter to
578 // kComplexityHigher, which maps to cpu_used = -4.
579 int pixels_per_frame = stream_codec->width * stream_codec->height;
580 if (pixels_per_frame < 352 * 288) {
581 stream_codec->VP8()->complexity =
582 webrtc::VideoCodecComplexity::kComplexityHigher;
583 }
584 // Turn off denoising for all streams but the highest resolution.
585 stream_codec->VP8()->denoisingOn = false;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000586 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000587 }
588 // TODO(ronghuawu): what to do with targetBitrate.
589
Erik Språng78ce6192016-09-12 16:04:43 +0200590 stream_codec->startBitrate = start_bitrate_kbps;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000591}
592
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000593bool SimulcastEncoderAdapter::Initialized() const {
brandtr5e171752017-05-23 03:32:16 -0700594 return rtc::AtomicOps::AcquireLoad(&inited_) == 1;
595}
596
597void SimulcastEncoderAdapter::DestroyStoredEncoders() {
598 while (!stored_encoders_.empty()) {
brandtr5e171752017-05-23 03:32:16 -0700599 stored_encoders_.pop();
600 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000601}
602
Erik Språng9b5b0702018-11-01 14:52:30 +0100603VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
Erik Språng75de46a2018-11-07 14:53:32 +0100604 return encoder_info_;
pbosecd21b42016-01-07 08:03:05 -0800605}
606
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000607} // namespace webrtc