blob: 78eeef288a2e7046c00adc934d438769ab9177a3 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
sprang3911c262016-04-15 01:24:14 -07002* Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3*
4* Use of this source code is governed by a BSD-style license
5* that can be found in the LICENSE file in the root of the source
6* tree. An additional intellectual property rights grant can be found
7* in the file PATENTS. All contributing project authors may
8* be found in the AUTHORS file in the root of the source tree.
9*/
niklase@google.com470e71d2011-07-07 08:21:25 +000010
philipel9d3ab612015-12-21 04:12:39 -080011#include "webrtc/modules/video_coding/generic_encoder.h"
12
13#include <vector>
14
nisseaf916892017-01-10 07:44:26 -080015#include "webrtc/api/video/i420_buffer.h"
Henrik Kjellander2557b862015-11-18 22:00:21 +010016#include "webrtc/modules/video_coding/encoded_frame.h"
Henrik Kjellander2557b862015-11-18 22:00:21 +010017#include "webrtc/modules/video_coding/media_optimization.h"
Edward Lemurc20978e2017-07-06 19:44:34 +020018#include "webrtc/rtc_base/checks.h"
19#include "webrtc/rtc_base/logging.h"
ilnik29d08402017-07-11 08:08:12 -070020#include "webrtc/rtc_base/optional.h"
Edward Lemurc20978e2017-07-06 19:44:34 +020021#include "webrtc/rtc_base/timeutils.h"
22#include "webrtc/rtc_base/trace_event.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000023
24namespace webrtc {
Sergey Ulanov525df3f2016-08-02 17:46:41 -070025
Peter Boström4f5db112015-10-29 16:53:59 +010026VCMGenericEncoder::VCMGenericEncoder(
27 VideoEncoder* encoder,
Peter Boström4f5db112015-10-29 16:53:59 +010028 VCMEncodedFrameCallback* encoded_frame_callback,
sprang3911c262016-04-15 01:24:14 -070029 bool internal_source)
pbos@webrtc.org891d4832015-02-26 13:15:22 +000030 : encoder_(encoder),
Peter Boström4f5db112015-10-29 16:53:59 +010031 vcm_encoded_frame_callback_(encoded_frame_callback),
sprang3911c262016-04-15 01:24:14 -070032 internal_source_(internal_source),
Erik Språng08127a92016-11-16 16:41:30 +010033 encoder_params_({BitrateAllocation(), 0, 0, 0}),
ilnik04f4d122017-06-19 07:18:55 -070034 is_screenshare_(false),
35 streams_or_svc_num_(0) {}
niklase@google.com470e71d2011-07-07 08:21:25 +000036
Peter Boström4f5db112015-10-29 16:53:59 +010037VCMGenericEncoder::~VCMGenericEncoder() {}
38
39int32_t VCMGenericEncoder::Release() {
Peter Boström02bafc62016-07-01 12:45:15 +020040 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
Peter Boström4fd6cda2016-01-26 10:19:53 +010041 TRACE_EVENT0("webrtc", "VCMGenericEncoder::Release");
Peter Boström4f5db112015-10-29 16:53:59 +010042 return encoder_->Release();
niklase@google.com470e71d2011-07-07 08:21:25 +000043}
44
Peter Boström4f5db112015-10-29 16:53:59 +010045int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings,
sprang3911c262016-04-15 01:24:14 -070046 int32_t number_of_cores,
47 size_t max_payload_size) {
Peter Boström02bafc62016-07-01 12:45:15 +020048 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
pbosd9eec762015-11-17 06:03:43 -080049 TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode");
Peter Boström4f5db112015-10-29 16:53:59 +010050 is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing;
ilnik04f4d122017-06-19 07:18:55 -070051 streams_or_svc_num_ = settings->numberOfSimulcastStreams;
52 if (settings->codecType == kVideoCodecVP9) {
53 streams_or_svc_num_ = settings->VP9().numberOfSpatialLayers;
54 }
55 if (streams_or_svc_num_ == 0)
56 streams_or_svc_num_ = 1;
57
58 vcm_encoded_frame_callback_->SetTimingFramesThresholds(
59 settings->timing_frame_thresholds);
60 vcm_encoded_frame_callback_->OnFrameRateChanged(settings->maxFramerate);
61
sprang3911c262016-04-15 01:24:14 -070062 if (encoder_->InitEncode(settings, number_of_cores, max_payload_size) != 0) {
Peter Boström4f5db112015-10-29 16:53:59 +010063 LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
64 "payload name: "
65 << settings->plName;
66 return -1;
67 }
68 encoder_->RegisterEncodeCompleteCallback(vcm_encoded_frame_callback_);
69 return 0;
niklase@google.com470e71d2011-07-07 08:21:25 +000070}
71
sprang3911c262016-04-15 01:24:14 -070072int32_t VCMGenericEncoder::Encode(const VideoFrame& frame,
73 const CodecSpecificInfo* codec_specific,
74 const std::vector<FrameType>& frame_types) {
Peter Boström02bafc62016-07-01 12:45:15 +020075 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
pbosd9eec762015-11-17 06:03:43 -080076 TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
sprang3911c262016-04-15 01:24:14 -070077 frame.timestamp());
pbosd9eec762015-11-17 06:03:43 -080078
sprang3911c262016-04-15 01:24:14 -070079 for (FrameType frame_type : frame_types)
pbos22993e12015-10-19 02:39:06 -070080 RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta);
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +000081
ilnik04f4d122017-06-19 07:18:55 -070082 for (size_t i = 0; i < streams_or_svc_num_; ++i)
83 vcm_encoded_frame_callback_->OnEncodeStarted(frame.render_time_ms(), i);
sprang3911c262016-04-15 01:24:14 -070084 int32_t result = encoder_->Encode(frame, codec_specific, &frame_types);
Peter Boströmb7d9a972015-12-18 16:01:11 +010085
Erik Språng2c4c9142015-06-24 11:24:44 +020086 if (is_screenshare_ &&
87 result == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) {
88 // Target bitrate exceeded, encoder state has been reset - try again.
sprang3911c262016-04-15 01:24:14 -070089 return encoder_->Encode(frame, codec_specific, &frame_types);
Erik Språng2c4c9142015-06-24 11:24:44 +020090 }
91
92 return result;
niklase@google.com470e71d2011-07-07 08:21:25 +000093}
94
Peter Boström69ccb332015-10-29 16:30:23 +010095void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) {
Peter Boström02bafc62016-07-01 12:45:15 +020096 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
Peter Boström69ccb332015-10-29 16:30:23 +010097 bool channel_parameters_have_changed;
98 bool rates_have_changed;
99 {
100 rtc::CritScope lock(&params_lock_);
101 channel_parameters_have_changed =
102 params.loss_rate != encoder_params_.loss_rate ||
103 params.rtt != encoder_params_.rtt;
104 rates_have_changed =
105 params.target_bitrate != encoder_params_.target_bitrate ||
106 params.input_frame_rate != encoder_params_.input_frame_rate;
107 encoder_params_ = params;
108 }
Erik Språng08127a92016-11-16 16:41:30 +0100109 if (channel_parameters_have_changed) {
110 int res = encoder_->SetChannelParameters(params.loss_rate, params.rtt);
111 if (res != 0) {
112 LOG(LS_WARNING) << "Error set encoder parameters (loss = "
113 << params.loss_rate << ", rtt = " << params.rtt
114 << "): " << res;
115 }
116 }
Peter Boström69ccb332015-10-29 16:30:23 +0100117 if (rates_have_changed) {
Erik Språng08127a92016-11-16 16:41:30 +0100118 int res = encoder_->SetRateAllocation(params.target_bitrate,
119 params.input_frame_rate);
120 if (res != 0) {
121 LOG(LS_WARNING) << "Error set encoder rate (total bitrate bps = "
122 << params.target_bitrate.get_sum_bps()
123 << ", framerate = " << params.input_frame_rate
124 << "): " << res;
125 }
ilnik04f4d122017-06-19 07:18:55 -0700126 vcm_encoded_frame_callback_->OnFrameRateChanged(params.input_frame_rate);
127 for (size_t i = 0; i < streams_or_svc_num_; ++i) {
128 size_t layer_bitrate_bytes_per_sec =
129 params.target_bitrate.GetSpatialLayerSum(i) / 8;
130 // VP9 rate control is not yet moved out of VP9Impl. Due to that rates
131 // are not split among spatial layers.
132 if (layer_bitrate_bytes_per_sec == 0)
133 layer_bitrate_bytes_per_sec = params.target_bitrate.get_sum_bps() / 8;
134 vcm_encoded_frame_callback_->OnTargetBitrateChanged(
135 layer_bitrate_bytes_per_sec, i);
136 }
Peter Boström69ccb332015-10-29 16:30:23 +0100137 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000138}
139
Peter Boström69ccb332015-10-29 16:30:23 +0100140EncoderParameters VCMGenericEncoder::GetEncoderParameters() const {
141 rtc::CritScope lock(&params_lock_);
142 return encoder_params_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000143}
144
philipel9d3ab612015-12-21 04:12:39 -0800145int32_t VCMGenericEncoder::SetPeriodicKeyFrames(bool enable) {
Peter Boström02bafc62016-07-01 12:45:15 +0200146 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
philipel9d3ab612015-12-21 04:12:39 -0800147 return encoder_->SetPeriodicKeyFrames(enable);
niklase@google.com470e71d2011-07-07 08:21:25 +0000148}
149
pbos@webrtc.org7b859cc2013-04-02 15:54:38 +0000150int32_t VCMGenericEncoder::RequestFrame(
stefan@webrtc.orgcf216862012-10-25 11:29:51 +0000151 const std::vector<FrameType>& frame_types) {
Peter Boström02bafc62016-07-01 12:45:15 +0200152 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
nissedf2ceb82016-12-15 06:29:53 -0800153
ilnik04f4d122017-06-19 07:18:55 -0700154 for (size_t i = 0; i < streams_or_svc_num_; ++i)
155 vcm_encoded_frame_callback_->OnEncodeStarted(0, i);
nissedf2ceb82016-12-15 06:29:53 -0800156 // TODO(nisse): Used only with internal source. Delete as soon as
157 // that feature is removed. The only implementation I've been able
nisse01d5a0b2017-05-31 06:33:21 -0700158 // to find ignores what's in the frame. With one exception: It seems
159 // a few test cases, e.g.,
160 // VideoSendStreamTest.VideoSendStreamStopSetEncoderRateToZero, set
161 // internal_source to true and use FakeEncoder. And the latter will
162 // happily encode this 1x1 frame and pass it on down the pipeline.
nissedf2ceb82016-12-15 06:29:53 -0800163 return encoder_->Encode(VideoFrame(I420Buffer::Create(1, 1),
164 kVideoRotation_0, 0),
165 NULL, &frame_types);
166 return 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000167}
168
philipel9d3ab612015-12-21 04:12:39 -0800169bool VCMGenericEncoder::InternalSource() const {
170 return internal_source_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000171}
172
Peter Boströmeb66e802015-06-05 11:08:03 +0200173bool VCMGenericEncoder::SupportsNativeHandle() const {
Peter Boström02bafc62016-07-01 12:45:15 +0200174 RTC_DCHECK_RUNS_SERIALIZED(&race_checker_);
Peter Boströmeb66e802015-06-05 11:08:03 +0200175 return encoder_->SupportsNativeHandle();
176}
177
andresp@webrtc.org1df9dc32014-01-09 08:01:57 +0000178VCMEncodedFrameCallback::VCMEncodedFrameCallback(
perkj376b1922016-05-02 11:35:24 -0700179 EncodedImageCallback* post_encode_callback,
180 media_optimization::MediaOptimization* media_opt)
181 : internal_source_(false),
182 post_encode_callback_(post_encode_callback),
ilnik04f4d122017-06-19 07:18:55 -0700183 media_opt_(media_opt),
184 framerate_(1),
185 last_timing_frame_time_ms_(-1),
186 timing_frames_thresholds_({-1, 0}) {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000187
sprang3911c262016-04-15 01:24:14 -0700188VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000189
ilnik04f4d122017-06-19 07:18:55 -0700190void VCMEncodedFrameCallback::OnTargetBitrateChanged(
191 size_t bitrate_bytes_per_second,
192 size_t simulcast_svc_idx) {
193 rtc::CritScope crit(&timing_params_lock_);
194 if (timing_frames_info_.size() < simulcast_svc_idx + 1)
195 timing_frames_info_.resize(simulcast_svc_idx + 1);
196 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec =
197 bitrate_bytes_per_second;
198}
199
200void VCMEncodedFrameCallback::OnFrameRateChanged(size_t framerate) {
201 rtc::CritScope crit(&timing_params_lock_);
202 framerate_ = framerate;
203}
204
205void VCMEncodedFrameCallback::OnEncodeStarted(int64_t capture_time_ms,
206 size_t simulcast_svc_idx) {
207 rtc::CritScope crit(&timing_params_lock_);
208 if (timing_frames_info_.size() < simulcast_svc_idx + 1)
209 timing_frames_info_.resize(simulcast_svc_idx + 1);
210 timing_frames_info_[simulcast_svc_idx].encode_start_time_ms[capture_time_ms] =
211 rtc::TimeMillis();
212}
213
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700214EncodedImageCallback::Result VCMEncodedFrameCallback::OnEncodedImage(
Peter Boströmb7d9a972015-12-18 16:01:11 +0100215 const EncodedImage& encoded_image,
sprang3911c262016-04-15 01:24:14 -0700216 const CodecSpecificInfo* codec_specific,
217 const RTPFragmentationHeader* fragmentation_header) {
pbosd9eec762015-11-17 06:03:43 -0800218 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
Peter Boströmb7d9a972015-12-18 16:01:11 +0100219 "timestamp", encoded_image._timeStamp);
ilnik04f4d122017-06-19 07:18:55 -0700220 size_t simulcast_svc_idx = 0;
221 if (codec_specific->codecType == kVideoCodecVP9) {
222 if (codec_specific->codecSpecific.VP9.num_spatial_layers > 1)
223 simulcast_svc_idx = codec_specific->codecSpecific.VP9.spatial_idx;
224 } else if (codec_specific->codecType == kVideoCodecVP8) {
225 simulcast_svc_idx = codec_specific->codecSpecific.VP8.simulcastIdx;
226 } else if (codec_specific->codecType == kVideoCodecGeneric) {
227 simulcast_svc_idx = codec_specific->codecSpecific.generic.simulcast_idx;
228 } else if (codec_specific->codecType == kVideoCodecH264) {
229 // TODO(ilnik): When h264 simulcast is landed, extract simulcast idx here.
230 }
231
ilnik29d08402017-07-11 08:08:12 -0700232 rtc::Optional<size_t> outlier_frame_size;
233 rtc::Optional<int64_t> encode_start_ms;
emircanf0f73782017-08-15 12:31:23 -0700234 bool is_timing_frame = false;
ilnik04f4d122017-06-19 07:18:55 -0700235 {
236 rtc::CritScope crit(&timing_params_lock_);
ilnik0b1e2f32017-07-10 12:25:29 -0700237
ilnik29d08402017-07-11 08:08:12 -0700238 // Encoders with internal sources do not call OnEncodeStarted and
239 // OnFrameRateChanged. |timing_frames_info_| may be not filled here.
240 if (simulcast_svc_idx < timing_frames_info_.size()) {
ilnik3635f442017-06-28 03:53:19 -0700241 auto encode_start_map =
242 &timing_frames_info_[simulcast_svc_idx].encode_start_time_ms;
243 auto it = encode_start_map->find(encoded_image.capture_time_ms_);
244 if (it != encode_start_map->end()) {
ilnik29d08402017-07-11 08:08:12 -0700245 encode_start_ms.emplace(it->second);
ilnik3635f442017-06-28 03:53:19 -0700246 // Assuming all encoders do not reorder frames within single stream,
247 // there may be some dropped frames with smaller timestamps. These
248 // should be purged.
249 encode_start_map->erase(encode_start_map->begin(), it);
250 encode_start_map->erase(it);
251 } else {
ilnik29d08402017-07-11 08:08:12 -0700252 // Encoder is with internal source: free our records of any frames just
253 // in case to free memory.
254 encode_start_map->clear();
ilnik3635f442017-06-28 03:53:19 -0700255 }
256
ilnik29d08402017-07-11 08:08:12 -0700257 size_t target_bitrate =
258 timing_frames_info_[simulcast_svc_idx].target_bitrate_bytes_per_sec;
259 if (framerate_ > 0 && target_bitrate > 0) {
260 // framerate and target bitrate were reported by encoder.
261 size_t average_frame_size = target_bitrate / framerate_;
262 outlier_frame_size.emplace(
263 average_frame_size *
264 timing_frames_thresholds_.outlier_ratio_percent / 100);
ilnik0b1e2f32017-07-10 12:25:29 -0700265 }
ilnik29d08402017-07-11 08:08:12 -0700266 }
267
268 // Check if it's time to send a timing frame.
269 int64_t timing_frame_delay_ms =
270 encoded_image.capture_time_ms_ - last_timing_frame_time_ms_;
271 // Trigger threshold if it's a first frame, too long passed since the last
272 // timing frame, or we already sent timing frame on a different simulcast
273 // stream with the same capture time.
274 if (last_timing_frame_time_ms_ == -1 ||
275 timing_frame_delay_ms >= timing_frames_thresholds_.delay_ms ||
276 timing_frame_delay_ms == 0) {
emircanf0f73782017-08-15 12:31:23 -0700277 is_timing_frame = true;
ilnik29d08402017-07-11 08:08:12 -0700278 last_timing_frame_time_ms_ = encoded_image.capture_time_ms_;
279 }
280
281 // Outliers trigger timing frames, but do not affect scheduled timing
282 // frames.
283 if (outlier_frame_size && encoded_image._length >= *outlier_frame_size) {
emircanf0f73782017-08-15 12:31:23 -0700284 is_timing_frame = true;
ilnik04f4d122017-06-19 07:18:55 -0700285 }
ilnik04f4d122017-06-19 07:18:55 -0700286 }
287
ilnik29d08402017-07-11 08:08:12 -0700288 // If encode start is not available that means that encoder uses internal
289 // source. In that case capture timestamp may be from a different clock with a
290 // drift relative to rtc::TimeMillis(). We can't use it for Timing frames,
291 // because to being sent in the network capture time required to be less than
292 // all the other timestamps.
emircanf0f73782017-08-15 12:31:23 -0700293 if (is_timing_frame && encode_start_ms) {
ilnik29d08402017-07-11 08:08:12 -0700294 encoded_image.SetEncodeTime(*encode_start_ms, rtc::TimeMillis());
ilnik04f4d122017-06-19 07:18:55 -0700295 }
296
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700297 Result result = post_encode_callback_->OnEncodedImage(
298 encoded_image, codec_specific, fragmentation_header);
299 if (result.error != Result::OK)
300 return result;
niklase@google.com470e71d2011-07-07 08:21:25 +0000301
sprang3911c262016-04-15 01:24:14 -0700302 if (media_opt_) {
303 media_opt_->UpdateWithEncodedData(encoded_image);
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700304 if (internal_source_) {
305 // Signal to encoder to drop next frame.
306 result.drop_next_frame = media_opt_->DropFrame();
307 }
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000308 }
Sergey Ulanov525df3f2016-08-02 17:46:41 -0700309 return result;
niklase@google.com470e71d2011-07-07 08:21:25 +0000310}
311
pbos@webrtc.orgd900e8b2013-07-03 15:12:26 +0000312} // namespace webrtc