blob: ccbf0d3fae14842ba7abfb2eafe50938fcd785f7 [file] [log] [blame]
pbos@webrtc.org9115cde2014-12-09 10:36:40 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "media/engine/simulcast_encoder_adapter.h"
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000012
Yves Gerey3e707812018-11-28 16:47:49 +010013#include <stdio.h>
14#include <string.h>
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000015#include <algorithm>
Yves Gerey3e707812018-11-28 16:47:49 +010016#include <cstdint>
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +020017#include <string>
18#include <utility>
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000019
Mirko Bonadeid9708072019-01-25 20:26:48 +010020#include "api/scoped_refptr.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010022#include "api/video/video_codec_constants.h"
Yves Gerey3e707812018-11-28 16:47:49 +010023#include "api/video/video_frame_buffer.h"
24#include "api/video/video_rotation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020025#include "api/video_codecs/video_encoder.h"
Magnus Jedvertdf4883d2017-11-17 14:44:55 +010026#include "api/video_codecs/video_encoder_factory.h"
Yves Gerey3e707812018-11-28 16:47:49 +010027#include "modules/video_coding/include/video_error_codes.h"
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +020028#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080029#include "rtc_base/atomic_ops.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "rtc_base/checks.h"
Erik Språng7f24fb92019-02-13 10:49:37 +010031#include "rtc_base/experiments/rate_control_settings.h"
Erik Språng16cb8f52019-04-12 13:59:09 +020032#include "rtc_base/logging.h"
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +020033#include "system_wrappers/include/field_trial.h"
Mirko Bonadei65432062017-12-11 09:32:13 +010034#include "third_party/libyuv/include/libyuv/scale.h"
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000035
36namespace {
37
38const unsigned int kDefaultMinQp = 2;
39const unsigned int kDefaultMaxQp = 56;
40// Max qp for lowest spatial resolution when doing simulcast.
41const unsigned int kLowestResMaxQp = 45;
42
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +020043absl::optional<unsigned int> GetScreenshareBoostedQpValue() {
44 std::string experiment_group =
45 webrtc::field_trial::FindFullName("WebRTC-BoostedScreenshareQp");
46 unsigned int qp;
47 if (sscanf(experiment_group.c_str(), "%u", &qp) != 1)
48 return absl::nullopt;
49 qp = std::min(qp, 63u);
50 qp = std::max(qp, 1u);
51 return qp;
52}
53
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000054uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
55 uint32_t bitrate_sum = 0;
56 for (int i = 0; i < streams; ++i) {
57 bitrate_sum += codec.simulcastStream[i].maxBitrate;
58 }
59 return bitrate_sum;
60}
61
62int NumberOfStreams(const webrtc::VideoCodec& codec) {
63 int streams =
64 codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
65 uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
66 if (simulcast_max_bitrate == 0) {
67 streams = 1;
68 }
69 return streams;
70}
71
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000072int VerifyCodec(const webrtc::VideoCodec* inst) {
brandtr5e171752017-05-23 03:32:16 -070073 if (inst == nullptr) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000074 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
75 }
76 if (inst->maxFramerate < 1) {
77 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
78 }
79 // allow zero to represent an unspecified maxBitRate
80 if (inst->maxBitrate > 0 && inst->startBitrate > inst->maxBitrate) {
81 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
82 }
83 if (inst->width <= 1 || inst->height <= 1) {
84 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
85 }
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +020086 if (inst->codecType == webrtc::kVideoCodecVP8 &&
87 inst->VP8().automaticResizeOn && inst->numberOfSimulcastStreams > 1) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +000088 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
89 }
90 return WEBRTC_VIDEO_CODEC_OK;
91}
92
Florent Castelli1b761ca2019-01-21 14:33:02 +010093bool StreamResolutionCompare(const webrtc::SimulcastStream& a,
94 const webrtc::SimulcastStream& b) {
95 return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
96 std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
97}
98
Noah Richards41ee1ea2015-04-15 09:24:26 -070099// An EncodedImageCallback implementation that forwards on calls to a
100// SimulcastEncoderAdapter, but with the stream index it's registered with as
101// the first parameter to Encoded.
102class AdapterEncodedImageCallback : public webrtc::EncodedImageCallback {
103 public:
104 AdapterEncodedImageCallback(webrtc::SimulcastEncoderAdapter* adapter,
105 size_t stream_idx)
106 : adapter_(adapter), stream_idx_(stream_idx) {}
107
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700108 EncodedImageCallback::Result OnEncodedImage(
109 const webrtc::EncodedImage& encoded_image,
110 const webrtc::CodecSpecificInfo* codec_specific_info,
111 const webrtc::RTPFragmentationHeader* fragmentation) override {
112 return adapter_->OnEncodedImage(stream_idx_, encoded_image,
113 codec_specific_info, fragmentation);
Noah Richards41ee1ea2015-04-15 09:24:26 -0700114 }
115
116 private:
117 webrtc::SimulcastEncoderAdapter* const adapter_;
118 const size_t stream_idx_;
119};
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000120} // namespace
121
122namespace webrtc {
123
Ilya Nikolaevskiy97b4ee52018-05-28 10:24:22 +0200124SimulcastEncoderAdapter::SimulcastEncoderAdapter(VideoEncoderFactory* factory,
125 const SdpVideoFormat& format)
Magnus Jedvertdf4883d2017-11-17 14:44:55 +0100126 : inited_(0),
127 factory_(factory),
Ilya Nikolaevskiy97b4ee52018-05-28 10:24:22 +0200128 video_format_(format),
Magnus Jedvertdf4883d2017-11-17 14:44:55 +0100129 encoded_complete_callback_(nullptr),
Erik Språng7f24fb92019-02-13 10:49:37 +0100130 experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
131 boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
132 .Vp8BoostBaseLayerQuality()) {
Rasmus Brandtc334ce92018-02-05 13:03:25 +0100133 RTC_DCHECK(factory_);
Erik Språng75de46a2018-11-07 14:53:32 +0100134 encoder_info_.implementation_name = "SimulcastEncoderAdapter";
Magnus Jedvertdf4883d2017-11-17 14:44:55 +0100135
brandtr5e171752017-05-23 03:32:16 -0700136 // The adapter is typically created on the worker thread, but operated on
137 // the encoder task queue.
138 encoder_queue_.Detach();
139
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000140 memset(&codec_, 0, sizeof(webrtc::VideoCodec));
141}
142
143SimulcastEncoderAdapter::~SimulcastEncoderAdapter() {
brandtr5e171752017-05-23 03:32:16 -0700144 RTC_DCHECK(!Initialized());
145 DestroyStoredEncoders();
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000146}
147
Elad Alon8f01c4e2019-06-28 15:19:43 +0200148void SimulcastEncoderAdapter::SetFecControllerOverride(
149 FecControllerOverride* fec_controller_override) {
150 // Ignored.
151}
152
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000153int SimulcastEncoderAdapter::Release() {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200154 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700155
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000156 while (!streaminfos_.empty()) {
magjed3f897582017-08-28 08:05:42 -0700157 std::unique_ptr<VideoEncoder> encoder =
158 std::move(streaminfos_.back().encoder);
brandtr5e171752017-05-23 03:32:16 -0700159 // Even though it seems very unlikely, there are no guarantees that the
Rasmus Brandt9cf9f752017-09-28 13:02:58 +0200160 // encoder will not call back after being Release()'d. Therefore, we first
161 // disable the callbacks here.
brandtr5e171752017-05-23 03:32:16 -0700162 encoder->RegisterEncodeCompleteCallback(nullptr);
Rasmus Brandt9cf9f752017-09-28 13:02:58 +0200163 encoder->Release();
brandtr5e171752017-05-23 03:32:16 -0700164 streaminfos_.pop_back(); // Deletes callback adapter.
magjed3f897582017-08-28 08:05:42 -0700165 stored_encoders_.push(std::move(encoder));
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000166 }
brandtr5e171752017-05-23 03:32:16 -0700167
168 // It's legal to move the encoder to another queue now.
169 encoder_queue_.Detach();
170
171 rtc::AtomicOps::ReleaseStore(&inited_, 0);
172
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000173 return WEBRTC_VIDEO_CODEC_OK;
174}
175
Elad Alon370f93a2019-06-11 14:57:57 +0200176// TODO(eladalon): s/inst/codec_settings/g.
177int SimulcastEncoderAdapter::InitEncode(
178 const VideoCodec* inst,
179 const VideoEncoder::Settings& settings) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200180 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700181
Elad Alon370f93a2019-06-11 14:57:57 +0200182 if (settings.number_of_cores < 1) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000183 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
184 }
185
186 int ret = VerifyCodec(inst);
187 if (ret < 0) {
188 return ret;
189 }
190
191 ret = Release();
192 if (ret < 0) {
193 return ret;
194 }
195
196 int number_of_streams = NumberOfStreams(*inst);
brandtr5e171752017-05-23 03:32:16 -0700197 RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
Peter Boströmd53c3892016-03-30 17:03:52 +0200198 const bool doing_simulcast = (number_of_streams > 1);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000199
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000200 codec_ = *inst;
Erik Språng82fad3d2018-03-21 09:57:23 +0100201 SimulcastRateAllocator rate_allocator(codec_);
Erik Språng566124a2018-04-23 12:32:22 +0200202 VideoBitrateAllocation allocation = rate_allocator.GetAllocation(
Erik Språng08127a92016-11-16 16:41:30 +0100203 codec_.startBitrate * 1000, codec_.maxFramerate);
204 std::vector<uint32_t> start_bitrates;
205 for (int i = 0; i < kMaxSimulcastStreams; ++i) {
206 uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
207 start_bitrates.push_back(stream_bitrate);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000208 }
209
Erik Språng75de46a2018-11-07 14:53:32 +0100210 encoder_info_.supports_native_handle = true;
211 encoder_info_.scaling_settings.thresholds = absl::nullopt;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000212 // Create |number_of_streams| of encoder instances and init them.
Florent Castelli1b761ca2019-01-21 14:33:02 +0100213
214 const auto minmax = std::minmax_element(
215 std::begin(codec_.simulcastStream),
216 std::begin(codec_.simulcastStream) + number_of_streams,
217 StreamResolutionCompare);
218 const auto lowest_resolution_stream_index =
219 std::distance(std::begin(codec_.simulcastStream), minmax.first);
220 const auto highest_resolution_stream_index =
221 std::distance(std::begin(codec_.simulcastStream), minmax.second);
222
223 RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
224 RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
225
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000226 for (int i = 0; i < number_of_streams; ++i) {
227 VideoCodec stream_codec;
Erik Språng78ce6192016-09-12 16:04:43 +0200228 uint32_t start_bitrate_kbps = start_bitrates[i];
Erik Språng7d687b12018-09-12 17:04:10 +0200229 const bool send_stream = start_bitrate_kbps > 0;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000230 if (!doing_simulcast) {
231 stream_codec = codec_;
232 stream_codec.numberOfSimulcastStreams = 1;
Erik Språng75de46a2018-11-07 14:53:32 +0100233
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000234 } else {
Erik Språng78ce6192016-09-12 16:04:43 +0200235 // Cap start bitrate to the min bitrate in order to avoid strange codec
Erik Språng7d687b12018-09-12 17:04:10 +0200236 // behavior. Since sending will be false, this should not matter.
Florent Castelli1b761ca2019-01-21 14:33:02 +0100237 StreamResolution stream_resolution =
238 i == highest_resolution_stream_index
239 ? StreamResolution::HIGHEST
240 : i == lowest_resolution_stream_index ? StreamResolution::LOWEST
241 : StreamResolution::OTHER;
242
Erik Språng78ce6192016-09-12 16:04:43 +0200243 start_bitrate_kbps =
244 std::max(codec_.simulcastStream[i].minBitrate, start_bitrate_kbps);
Florent Castelli1b761ca2019-01-21 14:33:02 +0100245 PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
246 &stream_codec);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000247 }
248
Erik Språngcc681cc2018-03-14 17:52:55 +0100249 // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000250 if (stream_codec.qpMax < kDefaultMinQp) {
251 stream_codec.qpMax = kDefaultMaxQp;
252 }
253
brandtr5e171752017-05-23 03:32:16 -0700254 // If an existing encoder instance exists, reuse it.
255 // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here,
256 // when we start storing that state outside the encoder wrappers.
magjed3f897582017-08-28 08:05:42 -0700257 std::unique_ptr<VideoEncoder> encoder;
brandtr5e171752017-05-23 03:32:16 -0700258 if (!stored_encoders_.empty()) {
magjed3f897582017-08-28 08:05:42 -0700259 encoder = std::move(stored_encoders_.top());
brandtr5e171752017-05-23 03:32:16 -0700260 stored_encoders_.pop();
261 } else {
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +0200262 encoder = factory_->CreateVideoEncoder(SdpVideoFormat(
263 codec_.codecType == webrtc::kVideoCodecVP8 ? "VP8" : "H264"));
brandtr5e171752017-05-23 03:32:16 -0700264 }
265
Elad Alon370f93a2019-06-11 14:57:57 +0200266 ret = encoder->InitEncode(&stream_codec, settings);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000267 if (ret < 0) {
noahrice5ba75a2016-12-12 13:08:27 -0800268 // Explicitly destroy the current encoder; because we haven't registered a
269 // StreamInfo for it yet, Release won't do anything about it.
magjed3f897582017-08-28 08:05:42 -0700270 encoder.reset();
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000271 Release();
272 return ret;
273 }
Erik Språngd3438aa2018-11-08 16:56:43 +0100274
brandtr5e171752017-05-23 03:32:16 -0700275 std::unique_ptr<EncodedImageCallback> callback(
276 new AdapterEncodedImageCallback(this, i));
277 encoder->RegisterEncodeCompleteCallback(callback.get());
magjed3f897582017-08-28 08:05:42 -0700278 streaminfos_.emplace_back(std::move(encoder), std::move(callback),
279 stream_codec.width, stream_codec.height,
Erik Språng7d687b12018-09-12 17:04:10 +0200280 send_stream);
brandtr5e171752017-05-23 03:32:16 -0700281
Erik Språng75de46a2018-11-07 14:53:32 +0100282 if (!doing_simulcast) {
283 // Without simulcast, just pass through the encoder info from the one
284 // active encoder.
285 encoder_info_ = streaminfos_[0].encoder->GetEncoderInfo();
286 } else {
287 const EncoderInfo encoder_impl_info =
288 streaminfos_[i].encoder->GetEncoderInfo();
289
290 if (i == 0) {
291 // Quality scaling not enabled for simulcast.
292 encoder_info_.scaling_settings = VideoEncoder::ScalingSettings::kOff;
293
294 // Encoder name indicates names of all sub-encoders.
295 encoder_info_.implementation_name = "SimulcastEncoderAdapter (";
296 encoder_info_.implementation_name +=
297 encoder_impl_info.implementation_name;
298
299 encoder_info_.supports_native_handle =
300 encoder_impl_info.supports_native_handle;
Erik Språngd3438aa2018-11-08 16:56:43 +0100301 encoder_info_.has_trusted_rate_controller =
302 encoder_impl_info.has_trusted_rate_controller;
Mirta Dvornicic897a9912018-11-30 13:12:21 +0100303 encoder_info_.is_hardware_accelerated =
304 encoder_impl_info.is_hardware_accelerated;
305 encoder_info_.has_internal_source =
306 encoder_impl_info.has_internal_source;
Erik Språng75de46a2018-11-07 14:53:32 +0100307 } else {
308 encoder_info_.implementation_name += ", ";
309 encoder_info_.implementation_name +=
310 encoder_impl_info.implementation_name;
311
312 // Native handle supported only if all encoders supports it.
313 encoder_info_.supports_native_handle &=
314 encoder_impl_info.supports_native_handle;
Erik Språngd3438aa2018-11-08 16:56:43 +0100315
316 // Trusted rate controller only if all encoders have it.
317 encoder_info_.has_trusted_rate_controller &=
318 encoder_impl_info.has_trusted_rate_controller;
Mirta Dvornicic897a9912018-11-30 13:12:21 +0100319
320 // Uses hardware support if any of the encoders uses it.
321 // For example, if we are having issues with down-scaling due to
322 // pipelining delay in HW encoders we need higher encoder usage
323 // thresholds in CPU adaptation.
324 encoder_info_.is_hardware_accelerated |=
325 encoder_impl_info.is_hardware_accelerated;
326
327 // Has internal source only if all encoders have it.
328 encoder_info_.has_internal_source &=
329 encoder_impl_info.has_internal_source;
Erik Språng75de46a2018-11-07 14:53:32 +0100330 }
Erik Språngdbdd8392019-01-17 15:27:50 +0100331 encoder_info_.fps_allocation[i] = encoder_impl_info.fps_allocation[0];
brandtr5e171752017-05-23 03:32:16 -0700332 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000333 }
brandtr5e171752017-05-23 03:32:16 -0700334
Peter Boströmd53c3892016-03-30 17:03:52 +0200335 if (doing_simulcast) {
Erik Språng75de46a2018-11-07 14:53:32 +0100336 encoder_info_.implementation_name += ")";
Peter Boströmd53c3892016-03-30 17:03:52 +0200337 }
brandtr5e171752017-05-23 03:32:16 -0700338
339 // To save memory, don't store encoders that we don't use.
340 DestroyStoredEncoders();
341
342 rtc::AtomicOps::ReleaseStore(&inited_, 1);
343
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000344 return WEBRTC_VIDEO_CODEC_OK;
345}
346
347int SimulcastEncoderAdapter::Encode(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700348 const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100349 const std::vector<VideoFrameType>* frame_types) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200350 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700351
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000352 if (!Initialized()) {
353 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
354 }
brandtr5e171752017-05-23 03:32:16 -0700355 if (encoded_complete_callback_ == nullptr) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000356 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
357 }
358
359 // All active streams should generate a key frame if
360 // a key frame is requested by any stream.
361 bool send_key_frame = false;
362 if (frame_types) {
363 for (size_t i = 0; i < frame_types->size(); ++i) {
Niels Möller8f7ce222019-03-21 15:43:58 +0100364 if (frame_types->at(i) == VideoFrameType::kVideoFrameKey) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000365 send_key_frame = true;
366 break;
367 }
368 }
369 }
370 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
371 if (streaminfos_[stream_idx].key_frame_request &&
372 streaminfos_[stream_idx].send_stream) {
373 send_key_frame = true;
374 break;
375 }
376 }
377
378 int src_width = input_image.width();
379 int src_height = input_image.height();
380 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
Peter Boström5d0379d2015-10-06 14:04:51 +0200381 // Don't encode frames in resolutions that we don't intend to send.
brandtr5e171752017-05-23 03:32:16 -0700382 if (!streaminfos_[stream_idx].send_stream) {
Peter Boström5d0379d2015-10-06 14:04:51 +0200383 continue;
brandtr5e171752017-05-23 03:32:16 -0700384 }
Peter Boström5d0379d2015-10-06 14:04:51 +0200385
Niels Möller87e2d782019-03-07 10:18:23 +0100386 std::vector<VideoFrameType> stream_frame_types;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000387 if (send_key_frame) {
Niels Möller8f7ce222019-03-21 15:43:58 +0100388 stream_frame_types.push_back(VideoFrameType::kVideoFrameKey);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000389 streaminfos_[stream_idx].key_frame_request = false;
390 } else {
Niels Möller8f7ce222019-03-21 15:43:58 +0100391 stream_frame_types.push_back(VideoFrameType::kVideoFrameDelta);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000392 }
393
394 int dst_width = streaminfos_[stream_idx].width;
395 int dst_height = streaminfos_[stream_idx].height;
396 // If scaling isn't required, because the input resolution
397 // matches the destination or the input image is empty (e.g.
398 // a keyframe request for encoders with internal camera
noahricfe3654d2016-07-01 09:05:54 -0700399 // sources) or the source image has a native handle, pass the image on
400 // directly. Otherwise, we'll scale it to match what the encoder expects
401 // (below).
402 // For texture frames, the underlying encoder is expected to be able to
403 // correctly sample/scale the source texture.
404 // TODO(perkj): ensure that works going forward, and figure out how this
405 // affects webrtc:5683.
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000406 if ((dst_width == src_width && dst_height == src_height) ||
Magnus Jedvert72dbe2a2017-06-10 17:03:37 +0000407 input_image.video_frame_buffer()->type() ==
408 VideoFrameBuffer::Type::kNative) {
Niels Möllerc8d2e732019-03-06 12:00:33 +0100409 int ret = streaminfos_[stream_idx].encoder->Encode(input_image,
410 &stream_frame_types);
noahric57779102016-05-25 06:48:46 -0700411 if (ret != WEBRTC_VIDEO_CODEC_OK) {
412 return ret;
413 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000414 } else {
nisse64ec8f82016-09-27 00:17:25 -0700415 rtc::scoped_refptr<I420Buffer> dst_buffer =
Magnus Jedvert72dbe2a2017-06-10 17:03:37 +0000416 I420Buffer::Create(dst_width, dst_height);
417 rtc::scoped_refptr<I420BufferInterface> src_buffer =
418 input_image.video_frame_buffer()->ToI420();
419 libyuv::I420Scale(src_buffer->DataY(), src_buffer->StrideY(),
420 src_buffer->DataU(), src_buffer->StrideU(),
421 src_buffer->DataV(), src_buffer->StrideV(), src_width,
422 src_height, dst_buffer->MutableDataY(),
423 dst_buffer->StrideY(), dst_buffer->MutableDataU(),
424 dst_buffer->StrideU(), dst_buffer->MutableDataV(),
425 dst_buffer->StrideV(), dst_width, dst_height,
nissec9c142f2016-05-17 04:05:47 -0700426 libyuv::kFilterBilinear);
nisse64ec8f82016-09-27 00:17:25 -0700427
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100428 // UpdateRect is not propagated to lower simulcast layers currently.
429 // TODO(ilnik): Consider scaling UpdateRect together with the buffer.
Ilya Nikolaevskiy4fc08552019-06-05 15:59:12 +0200430 VideoFrame frame(input_image);
431 frame.set_video_frame_buffer(dst_buffer);
432 frame.set_rotation(webrtc::kVideoRotation_0);
433 frame.set_update_rect(
434 VideoFrame::UpdateRect{0, 0, frame.width(), frame.height()});
Niels Möllerc8d2e732019-03-06 12:00:33 +0100435 int ret =
436 streaminfos_[stream_idx].encoder->Encode(frame, &stream_frame_types);
noahric57779102016-05-25 06:48:46 -0700437 if (ret != WEBRTC_VIDEO_CODEC_OK) {
438 return ret;
439 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000440 }
441 }
442
443 return WEBRTC_VIDEO_CODEC_OK;
444}
445
446int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
447 EncodedImageCallback* callback) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200448 RTC_DCHECK_RUN_ON(&encoder_queue_);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000449 encoded_complete_callback_ = callback;
450 return WEBRTC_VIDEO_CODEC_OK;
451}
452
Erik Språng16cb8f52019-04-12 13:59:09 +0200453void SimulcastEncoderAdapter::SetRates(
454 const RateControlParameters& parameters) {
Sebastian Janssonb55015e2019-04-09 13:44:04 +0200455 RTC_DCHECK_RUN_ON(&encoder_queue_);
brandtr5e171752017-05-23 03:32:16 -0700456
457 if (!Initialized()) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200458 RTC_LOG(LS_WARNING) << "SetRates while not initialized";
459 return;
brandtr5e171752017-05-23 03:32:16 -0700460 }
sprang647bf432016-11-10 06:46:20 -0800461
Erik Språng16cb8f52019-04-12 13:59:09 +0200462 if (parameters.framerate_fps < 1.0) {
463 RTC_LOG(LS_WARNING) << "Invalid framerate: " << parameters.framerate_fps;
464 return;
brandtr5e171752017-05-23 03:32:16 -0700465 }
Erik Språng08127a92016-11-16 16:41:30 +0100466
Erik Språng16cb8f52019-04-12 13:59:09 +0200467 if (codec_.maxBitrate > 0 &&
468 parameters.bitrate.get_sum_kbps() > codec_.maxBitrate) {
469 RTC_LOG(LS_WARNING) << "Total bitrate " << parameters.bitrate.get_sum_kbps()
470 << " exceeds max bitrate: " << codec_.maxBitrate;
471 return;
brandtr5e171752017-05-23 03:32:16 -0700472 }
Erik Språng08127a92016-11-16 16:41:30 +0100473
Erik Språng16cb8f52019-04-12 13:59:09 +0200474 if (parameters.bitrate.get_sum_bps() > 0) {
sprang1369c832016-11-10 08:30:33 -0800475 // Make sure the bitrate fits the configured min bitrates. 0 is a special
476 // value that means paused, though, so leave it alone.
Erik Språng16cb8f52019-04-12 13:59:09 +0200477 if (parameters.bitrate.get_sum_kbps() < codec_.minBitrate) {
478 RTC_LOG(LS_WARNING) << "Total bitrate "
479 << parameters.bitrate.get_sum_kbps()
480 << " is lower than minimum bitrate: "
481 << codec_.minBitrate;
482 return;
brandtr5e171752017-05-23 03:32:16 -0700483 }
Erik Språng08127a92016-11-16 16:41:30 +0100484
sprang1369c832016-11-10 08:30:33 -0800485 if (codec_.numberOfSimulcastStreams > 0 &&
Erik Språng16cb8f52019-04-12 13:59:09 +0200486 parameters.bitrate.get_sum_kbps() <
487 codec_.simulcastStream[0].minBitrate) {
488 RTC_LOG(LS_WARNING) << "Total bitrate "
489 << parameters.bitrate.get_sum_kbps()
490 << " is lower than minimum bitrate of base layer: "
491 << codec_.simulcastStream[0].minBitrate;
492 return;
sprang1369c832016-11-10 08:30:33 -0800493 }
sprang1369c832016-11-10 08:30:33 -0800494 }
Erik Språng08127a92016-11-16 16:41:30 +0100495
Erik Språng16cb8f52019-04-12 13:59:09 +0200496 codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000497
sprang1369c832016-11-10 08:30:33 -0800498 for (size_t stream_idx = 0; stream_idx < streaminfos_.size(); ++stream_idx) {
Erik Språng08127a92016-11-16 16:41:30 +0100499 uint32_t stream_bitrate_kbps =
Erik Språng16cb8f52019-04-12 13:59:09 +0200500 parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
Erik Språng08127a92016-11-16 16:41:30 +0100501
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000502 // Need a key frame if we have not sent this stream before.
Erik Språng78ce6192016-09-12 16:04:43 +0200503 if (stream_bitrate_kbps > 0 && !streaminfos_[stream_idx].send_stream) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000504 streaminfos_[stream_idx].key_frame_request = true;
505 }
Erik Språng78ce6192016-09-12 16:04:43 +0200506 streaminfos_[stream_idx].send_stream = stream_bitrate_kbps > 0;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000507
Erik Språng08127a92016-11-16 16:41:30 +0100508 // Slice the temporal layers out of the full allocation and pass it on to
509 // the encoder handling the current simulcast stream.
Erik Språng16cb8f52019-04-12 13:59:09 +0200510 RateControlParameters stream_parameters = parameters;
511 stream_parameters.bitrate = VideoBitrateAllocation();
brandtr5e171752017-05-23 03:32:16 -0700512 for (int i = 0; i < kMaxTemporalStreams; ++i) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200513 if (parameters.bitrate.HasBitrate(stream_idx, i)) {
514 stream_parameters.bitrate.SetBitrate(
515 0, i, parameters.bitrate.GetBitrate(stream_idx, i));
erikvarga@webrtc.org01f2ec32017-11-15 14:58:23 +0100516 }
brandtr5e171752017-05-23 03:32:16 -0700517 }
Minyue Li7ddef1a2019-04-11 10:50:19 +0000518
Erik Språng16cb8f52019-04-12 13:59:09 +0200519 // Assign link allocation proportionally to spatial layer allocation.
520 if (parameters.bandwidth_allocation != DataRate::Zero()) {
521 stream_parameters.bandwidth_allocation =
522 DataRate::bps((parameters.bandwidth_allocation.bps() *
523 stream_parameters.bitrate.get_sum_bps()) /
524 parameters.bitrate.get_sum_bps());
525 // Make sure we don't allocate bandwidth lower than target bitrate.
526 if (stream_parameters.bandwidth_allocation.bps() <
527 stream_parameters.bitrate.get_sum_bps()) {
528 stream_parameters.bandwidth_allocation =
529 DataRate::bps(stream_parameters.bitrate.get_sum_bps());
530 }
531 }
532
533 streaminfos_[stream_idx].encoder->SetRates(stream_parameters);
534 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000535}
536
Elad Alon65764e42019-06-28 18:43:44 +0200537void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
538 for (StreamInfo& info : streaminfos_) {
539 info.encoder->OnPacketLossRateUpdate(packet_loss_rate);
540 }
541}
542
543void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
544 for (StreamInfo& info : streaminfos_) {
545 info.encoder->OnRttUpdate(rtt_ms);
546 }
547}
548
549void SimulcastEncoderAdapter::OnLossNotification(
550 const LossNotification& loss_notification) {
551 for (StreamInfo& info : streaminfos_) {
552 info.encoder->OnLossNotification(loss_notification);
553 }
554}
555
brandtr5e171752017-05-23 03:32:16 -0700556// TODO(brandtr): Add task checker to this member function, when all encoder
557// callbacks are coming in on the encoder queue.
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700558EncodedImageCallback::Result SimulcastEncoderAdapter::OnEncodedImage(
Noah Richards41ee1ea2015-04-15 09:24:26 -0700559 size_t stream_idx,
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000560 const EncodedImage& encodedImage,
561 const CodecSpecificInfo* codecSpecificInfo,
562 const RTPFragmentationHeader* fragmentation) {
Niels Möllerd3b8c632018-08-27 15:33:42 +0200563 EncodedImage stream_image(encodedImage);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000564 CodecSpecificInfo stream_codec_specific = *codecSpecificInfo;
Niels Möllerd3b8c632018-08-27 15:33:42 +0200565
566 stream_image.SetSpatialIndex(stream_idx);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000567
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700568 return encoded_complete_callback_->OnEncodedImage(
Niels Möllerd3b8c632018-08-27 15:33:42 +0200569 stream_image, &stream_codec_specific, fragmentation);
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000570}
571
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000572void SimulcastEncoderAdapter::PopulateStreamCodec(
brandtr5e171752017-05-23 03:32:16 -0700573 const webrtc::VideoCodec& inst,
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000574 int stream_index,
Erik Språng78ce6192016-09-12 16:04:43 +0200575 uint32_t start_bitrate_kbps,
Florent Castelli1b761ca2019-01-21 14:33:02 +0100576 StreamResolution stream_resolution,
Erik Språng78ce6192016-09-12 16:04:43 +0200577 webrtc::VideoCodec* stream_codec) {
brandtr5e171752017-05-23 03:32:16 -0700578 *stream_codec = inst;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000579
580 // Stream specific settings.
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000581 stream_codec->numberOfSimulcastStreams = 0;
brandtr5e171752017-05-23 03:32:16 -0700582 stream_codec->width = inst.simulcastStream[stream_index].width;
583 stream_codec->height = inst.simulcastStream[stream_index].height;
584 stream_codec->maxBitrate = inst.simulcastStream[stream_index].maxBitrate;
585 stream_codec->minBitrate = inst.simulcastStream[stream_index].minBitrate;
586 stream_codec->qpMax = inst.simulcastStream[stream_index].qpMax;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000587 // Settings that are based on stream/resolution.
Florent Castelli1b761ca2019-01-21 14:33:02 +0100588 if (stream_resolution == StreamResolution::LOWEST) {
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000589 // Settings for lowest spatial resolutions.
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +0200590 if (inst.mode == VideoCodecMode::kScreensharing) {
591 if (experimental_boosted_screenshare_qp_) {
592 stream_codec->qpMax = *experimental_boosted_screenshare_qp_;
593 }
Erik Språng7f24fb92019-02-13 10:49:37 +0100594 } else if (boost_base_layer_quality_) {
Ilya Nikolaevskiyc30a1312018-08-27 11:07:48 +0200595 stream_codec->qpMax = kLowestResMaxQp;
596 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000597 }
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +0200598 if (inst.codecType == webrtc::kVideoCodecVP8) {
599 stream_codec->VP8()->numberOfTemporalLayers =
600 inst.simulcastStream[stream_index].numberOfTemporalLayers;
Florent Castelli1b761ca2019-01-21 14:33:02 +0100601 if (stream_resolution != StreamResolution::HIGHEST) {
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +0200602 // For resolutions below CIF, set the codec |complexity| parameter to
603 // kComplexityHigher, which maps to cpu_used = -4.
604 int pixels_per_frame = stream_codec->width * stream_codec->height;
605 if (pixels_per_frame < 352 * 288) {
606 stream_codec->VP8()->complexity =
607 webrtc::VideoCodecComplexity::kComplexityHigher;
608 }
609 // Turn off denoising for all streams but the highest resolution.
610 stream_codec->VP8()->denoisingOn = false;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000611 }
Johnny Leec5385062019-06-19 14:13:32 -0400612 } else if (inst.codecType == webrtc::kVideoCodecH264) {
613 stream_codec->H264()->numberOfTemporalLayers =
614 inst.simulcastStream[stream_index].numberOfTemporalLayers;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000615 }
616 // TODO(ronghuawu): what to do with targetBitrate.
617
Erik Språng78ce6192016-09-12 16:04:43 +0200618 stream_codec->startBitrate = start_bitrate_kbps;
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000619}
620
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000621bool SimulcastEncoderAdapter::Initialized() const {
brandtr5e171752017-05-23 03:32:16 -0700622 return rtc::AtomicOps::AcquireLoad(&inited_) == 1;
623}
624
625void SimulcastEncoderAdapter::DestroyStoredEncoders() {
626 while (!stored_encoders_.empty()) {
brandtr5e171752017-05-23 03:32:16 -0700627 stored_encoders_.pop();
628 }
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000629}
630
Erik Språng9b5b0702018-11-01 14:52:30 +0100631VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
Erik Språng75de46a2018-11-07 14:53:32 +0100632 return encoder_info_;
pbosecd21b42016-01-07 08:03:05 -0800633}
634
pbos@webrtc.org9115cde2014-12-09 10:36:40 +0000635} // namespace webrtc