blob: 5066ba153e17d4217abf7367b6e76e4649f906f7 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
sprang3911c262016-04-15 01:24:14 -07002* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3*
4* Use of this source code is governed by a BSD-style license
5* that can be found in the LICENSE file in the root of the source
6* tree. An additional intellectual property rights grant can be found
7* in the file PATENTS. All contributing project authors may
8* be found in the AUTHORS file in the root of the source tree.
9*/
niklase@google.com470e71d2011-07-07 08:21:25 +000010
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/generic_encoder.h"
philipel9d3ab612015-12-21 04:12:39 -080012
13#include <vector>
14
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020015#include "api/optional.h"
16#include "api/video/i420_buffer.h"
17#include "modules/pacing/alr_detector.h"
18#include "modules/video_coding/encoded_frame.h"
19#include "modules/video_coding/media_optimization.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
22#include "rtc_base/timeutils.h"
23#include "rtc_base/trace_event.h"
24#include "system_wrappers/include/field_trial.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000025
26namespace webrtc {
Sergey Ulanov525df3f2016-08-02 17:46:41 -070027
Peter Boström4f5db112015-10-29 16:53:59 +010028VCMGenericEncoder::VCMGenericEncoder(
29 VideoEncoder* encoder,
Peter Boström4f5db112015-10-29 16:53:59 +010030 VCMEncodedFrameCallback* encoded_frame_callback,
sprang3911c262016-04-15 01:24:14 -070031 bool internal_source)
pbos@webrtc.org891d4832015-02-26 13:15:22 +000032 : encoder_(encoder),
Peter Boström4f5db112015-10-29 16:53:59 +010033 vcm_encoded_frame_callback_(encoded_frame_callback),
sprang3911c262016-04-15 01:24:14 -070034 internal_source_(internal_source),
Erik Språng08127a92016-11-16 16:41:30 +010035 encoder_params_({BitrateAllocation(), 0, 0, 0}),
ilnik04f4d122017-06-19 07:18:55 -070036 streams_or_svc_num_(0) {}
niklase@google.com470e71d2011-07-07 08:21:25 +000037
Peter Boström4f5db112015-10-29 16:53:59 +010038VCMGenericEncoder::~VCMGenericEncoder() {}
39
40int32_t VCMGenericEncoder::Release() {
Peter Boström02bafc62016-07-01 12:45:15 +020041 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
Peter Boström4fd6cda2016-01-26 10:19:53 +010042 TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release");
Peter Boström4f5db112015-10-29 16:53:59 +010043 return encoder_->Release();
niklase@google.com470e71d2011-07-07 08:21:25 +000044}
45
Peter Boström4f5db112015-10-29 16:53:59 +010046int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings,
sprang3911c262016-04-15 01:24:14 -070047 int32_t number_of_cores,
48 size_t max_payload_size) {
Peter Boström02bafc62016-07-01 12:45:15 +020049 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
pbosd9eec762015-11-17 06:03:43 -080050 TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode");
ilnik04f4d122017-06-19 07:18:55 -070051 streams_or_svc_num_ = settings->numberOfSimulcastStreams;
52 if (settings->codecType == kVideoCodecVP9) {
53 streams_or_svc_num_ = settings->VP9().numberOfSpatialLayers;
54 }
55 if (streams_or_svc_num_ == 0)
56 streams_or_svc_num_ = 1;
57
58 vcm_encoded_frame_callback_->SetTimingFramesThresholds(
59 settings->timing_frame_thresholds);
60 vcm_encoded_frame_callback_->OnFrameRateChanged(settings->maxFramerate);
61
sprang3911c262016-04-15 01:24:14 -070062 if (encoder_->InitEncode(settings, number_of_cores, max_payload_size) != 0) {
Peter Boström4f5db112015-10-29 16:53:59 +010063 LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
64 "payload name: "
65 << settings->plName;
66 return -1;
67 }
68 encoder_->RegisterEncodeCompleteCallback(vcm_encoded_frame_callback_);
69 return 0;
niklase@google.com470e71d2011-07-07 08:21:25 +000070}
71
sprang3911c262016-04-15 01:24:14 -070072int32_t VCMGenericEncoder::Encode(const VideoFrame& frame,
73 const CodecSpecificInfo* codec_specific,
74 const std::vector<FrameType>& frame_types) {
Peter Boström02bafc62016-07-01 12:45:15 +020075 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
pbosd9eec762015-11-17 06:03:43 -080076 TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
sprang3911c262016-04-15 01:24:14 -070077 frame.timestamp());
pbosd9eec762015-11-17 06:03:43 -080078
sprang3911c262016-04-15 01:24:14 -070079 for (FrameType frame_type : frame_types)
pbos22993e12015-10-19 02:39:06 -070080 RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta);
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +000081
ilnik04f4d122017-06-19 07:18:55 -070082 for (size_t i = 0; i < streams_or_svc_num_; ++i)
83 vcm_encoded_frame_callback_->OnEncodeStarted(frame.render_time_ms(), i);
Peter Boströmb7d9a972015-12-18 16:01:11 +010084
ilnike9973812017-09-12 10:24:46 -070085 return encoder_->Encode(frame, codec_specific, &frame_types);
niklase@google.com470e71d2011-07-07 08:21:25 +000086}
87
Peter Boström69ccb332015-10-29 16:30:23 +010088void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) {
Peter Boström02bafc62016-07-01 12:45:15 +020089 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
Peter Boström69ccb332015-10-29 16:30:23 +010090 bool channel_parameters_have_changed;
91 bool rates_have_changed;
92 {
93 rtc::CritScope lock(&params_lock_);
94 channel_parameters_have_changed =
95 params.loss_rate != encoder_params_.loss_rate ||
96 params.rtt != encoder_params_.rtt;
97 rates_have_changed =
98 params.target_bitrate != encoder_params_.target_bitrate ||
99 params.input_frame_rate != encoder_params_.input_frame_rate;
100 encoder_params_ = params;
101 }
Erik Språng08127a92016-11-16 16:41:30 +0100102 if (channel_parameters_have_changed) {
103 int res = encoder_->SetChannelParameters(params.loss_rate, params.rtt);
104 if (res != 0) {
105 LOG(LS_WARNING) << "Error set encoder parameters (loss = "
106 << params.loss_rate << ", rtt = " << params.rtt
107 << "): " << res;
108 }
109 }
Peter Boström69ccb332015-10-29 16:30:23 +0100110 if (rates_have_changed) {
Erik Språng08127a92016-11-16 16:41:30 +0100111 int res = encoder_->SetRateAllocation(params.target_bitrate,
112 params.input_frame_rate);
113 if (res != 0) {
114 LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
115 << params.target_bitrate.get_sum_bps()
116 << ", framerate = " << params.input_frame_rate
117 << "): " << res;
118 }
ilnik04f4d122017-06-19 07:18:55 -0700119 vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate);
120 for (size_t i = 0; i < streams_or_svc_num_; ++i) {
121 size_t layer_bitrate_bytes_per_sec =
122 params.target_bitrate.GetSpatialLayerSum(i) / 8;
123 // VP9 rate control is not yet moved out of VP9Impl. Due to that rates
124 // are not split among spatial layers.
125 if (layer_bitrate_bytes_per_sec == 0)
126 layer_bitrate_bytes_per_sec = params.target_bitrate.get_sum_bps() / 8;
127 vcm_encoded_frame_callback_->OnTargetBitrateChanged(
128 layer_bitrate_bytes_per_sec, i);
129 }
Peter Boström69ccb332015-10-29 16:30:23 +0100130 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000131}
132
Peter Boström69ccb332015-10-29 16:30:23 +0100133EncoderParameters VCMGenericEncoder::GetEncoderParameters() const {
134 rtc::CritScope lock(&params_lock_);
135 return encoder_params_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000136}
137
philipel9d3ab612015-12-21 04:12:39 -0800138int32_t VCMGenericEncoder::SetPeriodicKeyFrames(bool enable) {
Peter Boström02bafc62016-07-01 12:45:15 +0200139 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
philipel9d3ab612015-12-21 04:12:39 -0800140 return encoder_->SetPeriodicKeyFrames(enable);
niklase@google.com470e71d2011-07-07 08:21:25 +0000141}
142
pbos@webrtc.org7b859cc2013-04-02 15:54:38 +0000143int32_t VCMGenericEncoder::RequestFrame(
stefan@webrtc.orgcf216862012-10-25 11:29:51 +0000144 const std::vector<FrameType>& frame_types) {
Peter Boström02bafc62016-07-01 12:45:15 +0200145 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
nissedf2ceb82016-12-15 06:29:53 -0800146
ilnik04f4d122017-06-19 07:18:55 -0700147 for (size_t i = 0; i < streams_or_svc_num_; ++i)
148 vcm_encoded_frame_callback_->OnEncodeStarted(0, i);
nissedf2ceb82016-12-15 06:29:53 -0800149 // TODO(nisse): Used only with internal source. Delete as soon as
150 // that feature is removed. The only implementation I've been able
nisse01d5a0b2017-05-31 06:33:21 -0700151 // to find ignores what's in the frame. With one exception: It seems
152 // a few test cases, e.g.,
153 // VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set
154 // internal_source to true and use FakeEncoder. And the latter will
155 // happily encode this 1x1 frame and pass it on down the pipeline.
nissedf2ceb82016-12-15 06:29:53 -0800156 return encoder_->Encode(VideoFrame(I420Buffer::Create(1, 1),
157 kVideoRotation_0, 0),
158 NULL, &frame_types);
159 return 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000160}
161
philipel9d3ab612015-12-21 04:12:39 -0800162bool VCMGenericEncoder::InternalSource() const {
163 return internal_source_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000164}
165
Peter Boströmeb66e802015-06-05 11:08:03 +0200166bool VCMGenericEncoder::SupportsNativeHandle() const {
Peter Boström02bafc62016-07-01 12:45:15 +0200167 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
Peter Boströmeb66e802015-06-05 11:08:03 +0200168 return encoder_->SupportsNativeHandle();
169}
170
andresp@webrtc.org1df9dc32014-01-09 08:01:57 +0000171VCMEncodedFrameCallback::VCMEncodedFrameCallback(
perkj376b1922016-05-02 11:35:24 -0700172 EncodedImageCallback* post_encode_callback,
173 media_optimization::MediaOptimization* media_opt)
174 : internal_source_(false),
175 post_encode_callback_(post_encode_callback),
ilnik04f4d122017-06-19 07:18:55 -0700176 media_opt_(media_opt),
177 framerate_(1),
178 last_timing_frame_time_ms_(-1),
ilnik6d5b4d62017-08-30 03:32:14 -0700179 timing_frames_thresholds_({-1, 0}) {
180 rtc::Optional<AlrDetector::AlrExperimentSettings> experiment_settings =
181 AlrDetector::ParseAlrSettingsFromFieldTrial(
182 AlrDetector::kStrictPacingAndProbingExperimentName);
183 if (experiment_settings) {
184 experiment_groups_[0] = experiment_settings->group_id + 1;
185 } else {
186 experiment_groups_[0] = 0;
187 }
188 experiment_settings = AlrDetector::ParseAlrSettingsFromFieldTrial(
189 AlrDetector::kScreenshareProbingBweExperimentName);
190 if (experiment_settings) {
191 experiment_groups_[1] = experiment_settings->group_id + 1;
192 } else {
193 experiment_groups_[1] = 0;
194 }
195}
niklase@google.com470e71d2011-07-07 08:21:25 +0000196
sprang3911c262016-04-15 01:24:14 -0700197VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000198
ilnik04f4d122017-06-19 07:18:55 -0700199void VCMEncodedFrameCallback::OnTargetBitrateChanged(
200 size_t bitrate_bytes_per_second,
201 size_t simulcast_svc_idx) {
202 rtc::CritScope crit(&timing_params_lock_);
203 if (timing_frames_info_.size() < simulcast_svc_idx + 1)
204 timing_frames_info_.resize(simulcast_svc_idx + 1);
205 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec =
206 bitrate_bytes_per_second;
207}
208
209void VCMEncodedFrameCallback::OnFrameRateChanged(size_t framerate) {
210 rtc::CritScope crit(&timing_params_lock_);
211 framerate_ = framerate;
212}
213
214void VCMEncodedFrameCallback::OnEncodeStarted(int64_t capture_time_ms,
215 size_t simulcast_svc_idx) {
216 rtc::CritScope crit(&timing_params_lock_);
217 if (timing_frames_info_.size() < simulcast_svc_idx + 1)
218 timing_frames_info_.resize(simulcast_svc_idx + 1);
Ilya Nikolaevskiy4b1a3632017-10-19 11:36:02 +0200219 RTC_DCHECK(
220 timing_frames_info_[simulcast_svc_idx].encode_start_list.empty() ||
221 rtc::TimeDiff(capture_time_ms, timing_frames_info_[simulcast_svc_idx]
222 .encode_start_list.back()
223 .capture_time_ms) >= 0);
224 if (timing_frames_info_[simulcast_svc_idx].encode_start_list.size() ==
225 kMaxEncodeStartTimeListSize) {
226 LOG(LS_WARNING) << "Too many frames in the encode_start_list."
227 " Did encoder stall?";
228 post_encode_callback_->OnDroppedFrame(DropReason::kDroppedByEncoder);
229 timing_frames_info_[simulcast_svc_idx].encode_start_list.pop_front();
230 }
231 timing_frames_info_[simulcast_svc_idx].encode_start_list.emplace_back(
232 capture_time_ms, rtc::TimeMillis());
ilnik04f4d122017-06-19 07:18:55 -0700233}
234
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700235EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage(
Peter Boströmb7d9a972015-12-18 16:01:11 +0100236 const EncodedImage& encoded_image,
sprang3911c262016-04-15 01:24:14 -0700237 const CodecSpecificInfo* codec_specific,
238 const RTPFragmentationHeader* fragmentation_header) {
pbosd9eec762015-11-17 06:03:43 -0800239 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
Peter Boströmb7d9a972015-12-18 16:01:11 +0100240 "timestamp", encoded_image._timeStamp);
ilnik04f4d122017-06-19 07:18:55 -0700241 size_t simulcast_svc_idx = 0;
242 if (codec_specific->codecType == kVideoCodecVP9) {
243 if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1)
244 simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx;
245 } else if (codec_specific->codecType == kVideoCodecVP8) {
246 simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx;
247 } else if (codec_specific->codecType == kVideoCodecGeneric) {
248 simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx;
249 } else if (codec_specific->codecType == kVideoCodecH264) {
250 // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here.
251 }
252
ilnik29d08402017-07-11 08:08:12 -0700253 rtc::Optional<size_t> outlier_frame_size;
254 rtc::Optional<int64_t> encode_start_ms;
ilnik6d5b4d62017-08-30 03:32:14 -0700255 size_t num_simulcast_svc_streams = 1;
sprangba050a62017-08-18 02:51:12 -0700256 uint8_t timing_flags = TimingFrameFlags::kInvalid;
ilnik04f4d122017-06-19 07:18:55 -0700257 {
258 rtc::CritScope crit(&timing_params_lock_);
ilnik0b1e2f32017-07-10 12:25:29 -0700259
ilnik29d08402017-07-11 08:08:12 -0700260 // Encoders with internal sources do not call OnEncodeStarted and
261 // OnFrameRateChanged. |timing_frames_info_| may be not filled here.
ilnik6d5b4d62017-08-30 03:32:14 -0700262 num_simulcast_svc_streams = timing_frames_info_.size();
263 if (simulcast_svc_idx < num_simulcast_svc_streams) {
Ilya Nikolaevskiy4b1a3632017-10-19 11:36:02 +0200264 auto encode_start_list =
265 &timing_frames_info_[simulcast_svc_idx].encode_start_list;
266 // Skip frames for which there was OnEncodeStarted but no OnEncodedImage
267 // call. These are dropped by encoder internally.
268 while (!encode_start_list->empty() &&
269 encode_start_list->front().capture_time_ms <
270 encoded_image.capture_time_ms_) {
271 post_encode_callback_->OnDroppedFrame(DropReason::kDroppedByEncoder);
272 encode_start_list->pop_front();
273 }
274 if (encode_start_list->size() > 0 &&
275 encode_start_list->front().capture_time_ms ==
276 encoded_image.capture_time_ms_) {
277 encode_start_ms.emplace(
278 encode_start_list->front().encode_start_time_ms);
279 encode_start_list->pop_front();
ilnik3635f442017-06-28 03:53:19 -0700280 }
281
ilnik29d08402017-07-11 08:08:12 -0700282 size_t target_bitrate =
283 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec;
284 if (framerate_ > 0 && target_bitrate > 0) {
285 // framerate and target bitrate were reported by encoder.
286 size_t average_frame_size = target_bitrate / framerate_;
287 outlier_frame_size.emplace(
288 average_frame_size *
289 timing_frames_thresholds_.outlier_ratio_percent / 100);
ilnik0b1e2f32017-07-10 12:25:29 -0700290 }
ilnik29d08402017-07-11 08:08:12 -0700291 }
292
293 // Check if it's time to send a timing frame.
294 int64_t timing_frame_delay_ms =
295 encoded_image.capture_time_ms_ - last_timing_frame_time_ms_;
296 // Trigger threshold if it's a first frame, too long passed since the last
297 // timing frame, or we already sent timing frame on a different simulcast
298 // stream with the same capture time.
299 if (last_timing_frame_time_ms_ == -1 ||
300 timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms ||
301 timing_frame_delay_ms == 0) {
sprangba050a62017-08-18 02:51:12 -0700302 timing_flags = TimingFrameFlags::kTriggeredByTimer;
ilnik29d08402017-07-11 08:08:12 -0700303 last_timing_frame_time_ms_ = encoded_image.capture_time_ms_;
304 }
305
306 // Outliers trigger timing frames, but do not affect scheduled timing
307 // frames.
308 if (outlier_frame_size && encoded_image._length >= *outlier_frame_size) {
sprangba050a62017-08-18 02:51:12 -0700309 if (timing_flags == TimingFrameFlags::kInvalid)
310 timing_flags = 0;
311 timing_flags |= TimingFrameFlags::kTriggeredBySize;
ilnik04f4d122017-06-19 07:18:55 -0700312 }
ilnik04f4d122017-06-19 07:18:55 -0700313 }
314
ilnik29d08402017-07-11 08:08:12 -0700315 // If encode start is not available that means that encoder uses internal
316 // source. In that case capture timestamp may be from a different clock with a
317 // drift relative to rtc::TimeMillis(). We can't use it for Timing frames,
318 // because to being sent in the network capture time required to be less than
319 // all the other timestamps.
sprangba050a62017-08-18 02:51:12 -0700320 if (timing_flags != TimingFrameFlags::kInvalid && encode_start_ms) {
ilnik29d08402017-07-11 08:08:12 -0700321 encoded_image.SetEncodeTime(*encode_start_ms, rtc::TimeMillis());
sprangba050a62017-08-18 02:51:12 -0700322 encoded_image.timing_.flags = timing_flags;
323 } else {
324 encoded_image.timing_.flags = TimingFrameFlags::kInvalid;
ilnik04f4d122017-06-19 07:18:55 -0700325 }
326
ilnik6d5b4d62017-08-30 03:32:14 -0700327 // Piggyback ALR experiment group id and simulcast id into the content type.
328 uint8_t experiment_id =
329 experiment_groups_[videocontenttypehelpers::IsScreenshare(
330 encoded_image.content_type_)];
331
332 // TODO(ilnik): This will force content type extension to be present even
333 // for realtime video. At the expense of miniscule overhead we will get
334 // sliced receive statistics.
335 RTC_CHECK(videocontenttypehelpers::SetExperimentId(
336 &encoded_image.content_type_, experiment_id));
337 // We count simulcast streams from 1 on the wire. That's why we set simulcast
338 // id in content type to +1 of that is actual simulcast index. This is because
339 // value 0 on the wire is reserved for 'no simulcast stream specified'.
340 RTC_CHECK(videocontenttypehelpers::SetSimulcastId(
341 &encoded_image.content_type_,
342 static_cast<uint8_t>(simulcast_svc_idx + 1)));
343
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700344 Result result = post_encode_callback_->OnEncodedImage(
345 encoded_image, codec_specific, fragmentation_header);
346 if (result.error != Result::OK)
347 return result;
niklase@google.com470e71d2011-07-07 08:21:25 +0000348
sprang3911c262016-04-15 01:24:14 -0700349 if (media_opt_) {
350 media_opt_->UpdateWithEncodedData(encoded_image);
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700351 if (internal_source_) {
352 // Signal to encoder to drop next frame.
353 result.drop_next_frame = media_opt_->DropFrame();
354 }
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000355 }
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700356 return result;
niklase@google.com470e71d2011-07-07 08:21:25 +0000357}
358
pbos@webrtc.orgd900e8b2013-07-03 15:12:26 +0000359} // namespace webrtc