blob: c7444ce99f3bc2210545518f611dcb39fc8aa847 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
pwestin@webrtc.org52fd98d2012-02-13 09:03:53 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
philipel9d3ab612015-12-21 04:12:39 -080011#include "webrtc/modules/video_coding/generic_encoder.h"
12
13#include <vector>
14
Guo-wei Shieh2c370782015-04-08 13:00:10 -070015#include "webrtc/base/checks.h"
pbos854e84c2015-11-16 16:39:06 -080016#include "webrtc/base/logging.h"
pbosd9eec762015-11-17 06:03:43 -080017#include "webrtc/base/trace_event.h"
pbos@webrtc.orga4407322013-07-16 12:32:05 +000018#include "webrtc/engine_configurations.h"
Henrik Kjellander2557b862015-11-18 22:00:21 +010019#include "webrtc/modules/video_coding/encoded_frame.h"
Henrik Kjellander2557b862015-11-18 22:00:21 +010020#include "webrtc/modules/video_coding/media_optimization.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010021#include "webrtc/system_wrappers/include/critical_section_wrapper.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000022
23namespace webrtc {
andresp@webrtc.orgc5aeb2a2014-01-09 08:04:32 +000024namespace {
25// Map information from info into rtp. If no relevant information is found
26// in info, rtp is set to NULL.
Guo-wei Shieh2c370782015-04-08 13:00:10 -070027void CopyCodecSpecific(const CodecSpecificInfo* info, RTPVideoHeader* rtp) {
henrikg91d6ede2015-09-17 00:24:34 -070028 RTC_DCHECK(info);
andresp@webrtc.orgc5aeb2a2014-01-09 08:04:32 +000029 switch (info->codecType) {
30 case kVideoCodecVP8: {
Guo-wei Shieh2c370782015-04-08 13:00:10 -070031 rtp->codec = kRtpVideoVp8;
32 rtp->codecHeader.VP8.InitRTPVideoHeaderVP8();
33 rtp->codecHeader.VP8.pictureId = info->codecSpecific.VP8.pictureId;
philipel9d3ab612015-12-21 04:12:39 -080034 rtp->codecHeader.VP8.nonReference = info->codecSpecific.VP8.nonReference;
Guo-wei Shieh2c370782015-04-08 13:00:10 -070035 rtp->codecHeader.VP8.temporalIdx = info->codecSpecific.VP8.temporalIdx;
36 rtp->codecHeader.VP8.layerSync = info->codecSpecific.VP8.layerSync;
37 rtp->codecHeader.VP8.tl0PicIdx = info->codecSpecific.VP8.tl0PicIdx;
38 rtp->codecHeader.VP8.keyIdx = info->codecSpecific.VP8.keyIdx;
39 rtp->simulcastIdx = info->codecSpecific.VP8.simulcastIdx;
andresp@webrtc.orgc5aeb2a2014-01-09 08:04:32 +000040 return;
41 }
asaperssona9455ab2015-07-31 06:10:09 -070042 case kVideoCodecVP9: {
43 rtp->codec = kRtpVideoVp9;
44 rtp->codecHeader.VP9.InitRTPVideoHeaderVP9();
45 rtp->codecHeader.VP9.inter_pic_predicted =
46 info->codecSpecific.VP9.inter_pic_predicted;
47 rtp->codecHeader.VP9.flexible_mode =
48 info->codecSpecific.VP9.flexible_mode;
49 rtp->codecHeader.VP9.ss_data_available =
50 info->codecSpecific.VP9.ss_data_available;
51 rtp->codecHeader.VP9.picture_id = info->codecSpecific.VP9.picture_id;
52 rtp->codecHeader.VP9.tl0_pic_idx = info->codecSpecific.VP9.tl0_pic_idx;
53 rtp->codecHeader.VP9.temporal_idx = info->codecSpecific.VP9.temporal_idx;
54 rtp->codecHeader.VP9.spatial_idx = info->codecSpecific.VP9.spatial_idx;
55 rtp->codecHeader.VP9.temporal_up_switch =
56 info->codecSpecific.VP9.temporal_up_switch;
57 rtp->codecHeader.VP9.inter_layer_predicted =
58 info->codecSpecific.VP9.inter_layer_predicted;
59 rtp->codecHeader.VP9.gof_idx = info->codecSpecific.VP9.gof_idx;
ivica7f6a6fc2015-09-08 02:40:29 -070060 rtp->codecHeader.VP9.num_spatial_layers =
61 info->codecSpecific.VP9.num_spatial_layers;
philipelcfc319b2015-11-10 07:17:23 -080062
asaperssona9455ab2015-07-31 06:10:09 -070063 if (info->codecSpecific.VP9.ss_data_available) {
asaperssona9455ab2015-07-31 06:10:09 -070064 rtp->codecHeader.VP9.spatial_layer_resolution_present =
65 info->codecSpecific.VP9.spatial_layer_resolution_present;
66 if (info->codecSpecific.VP9.spatial_layer_resolution_present) {
67 for (size_t i = 0; i < info->codecSpecific.VP9.num_spatial_layers;
68 ++i) {
69 rtp->codecHeader.VP9.width[i] = info->codecSpecific.VP9.width[i];
70 rtp->codecHeader.VP9.height[i] = info->codecSpecific.VP9.height[i];
71 }
72 }
73 rtp->codecHeader.VP9.gof.CopyGofInfoVP9(info->codecSpecific.VP9.gof);
74 }
philipelcfc319b2015-11-10 07:17:23 -080075
76 rtp->codecHeader.VP9.num_ref_pics = info->codecSpecific.VP9.num_ref_pics;
77 for (int i = 0; i < info->codecSpecific.VP9.num_ref_pics; ++i)
78 rtp->codecHeader.VP9.pid_diff[i] = info->codecSpecific.VP9.p_diff[i];
asaperssona9455ab2015-07-31 06:10:09 -070079 return;
80 }
stefan@webrtc.org2ec56062014-07-31 14:59:24 +000081 case kVideoCodecH264:
Guo-wei Shieh2c370782015-04-08 13:00:10 -070082 rtp->codec = kRtpVideoH264;
stefan@webrtc.org2ec56062014-07-31 14:59:24 +000083 return;
andresp@webrtc.orgc5aeb2a2014-01-09 08:04:32 +000084 case kVideoCodecGeneric:
Guo-wei Shieh2c370782015-04-08 13:00:10 -070085 rtp->codec = kRtpVideoGeneric;
86 rtp->simulcastIdx = info->codecSpecific.generic.simulcast_idx;
andresp@webrtc.orgc5aeb2a2014-01-09 08:04:32 +000087 return;
88 default:
andresp@webrtc.orgc5aeb2a2014-01-09 08:04:32 +000089 return;
90 }
91}
92} // namespace
niklase@google.com470e71d2011-07-07 08:21:25 +000093
philipel9d3ab612015-12-21 04:12:39 -080094// #define DEBUG_ENCODER_BIT_STREAM
niklase@google.com470e71d2011-07-07 08:21:25 +000095
Peter Boström4f5db112015-10-29 16:53:59 +010096VCMGenericEncoder::VCMGenericEncoder(
97 VideoEncoder* encoder,
98 VideoEncoderRateObserver* rate_observer,
99 VCMEncodedFrameCallback* encoded_frame_callback,
100 bool internalSource)
pbos@webrtc.org891d4832015-02-26 13:15:22 +0000101 : encoder_(encoder),
102 rate_observer_(rate_observer),
Peter Boström4f5db112015-10-29 16:53:59 +0100103 vcm_encoded_frame_callback_(encoded_frame_callback),
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000104 internal_source_(internalSource),
Peter Boström4f5db112015-10-29 16:53:59 +0100105 encoder_params_({0, 0, 0, 0}),
Erik Språng2c4c9142015-06-24 11:24:44 +0200106 rotation_(kVideoRotation_0),
Peter Boström69ccb332015-10-29 16:30:23 +0100107 is_screenshare_(false) {}
niklase@google.com470e71d2011-07-07 08:21:25 +0000108
Peter Boström4f5db112015-10-29 16:53:59 +0100109VCMGenericEncoder::~VCMGenericEncoder() {}
110
111int32_t VCMGenericEncoder::Release() {
112 return encoder_->Release();
niklase@google.com470e71d2011-07-07 08:21:25 +0000113}
114
Peter Boström4f5db112015-10-29 16:53:59 +0100115int32_t VCMGenericEncoder::InitEncode(const VideoCodec* settings,
116 int32_t numberOfCores,
117 size_t maxPayloadSize) {
pbosd9eec762015-11-17 06:03:43 -0800118 TRACE_EVENT0("webrtc", "VCMGenericEncoder::InitEncode");
Peter Boström4f5db112015-10-29 16:53:59 +0100119 {
120 rtc::CritScope lock(&params_lock_);
121 encoder_params_.target_bitrate = settings->startBitrate * 1000;
122 encoder_params_.input_frame_rate = settings->maxFramerate;
123 }
tommi@webrtc.org558dc402015-03-07 20:55:56 +0000124
Peter Boström4f5db112015-10-29 16:53:59 +0100125 is_screenshare_ = settings->mode == VideoCodecMode::kScreensharing;
126 if (encoder_->InitEncode(settings, numberOfCores, maxPayloadSize) != 0) {
127 LOG(LS_ERROR) << "Failed to initialize the encoder associated with "
128 "payload name: "
129 << settings->plName;
130 return -1;
131 }
132 encoder_->RegisterEncodeCompleteCallback(vcm_encoded_frame_callback_);
133 return 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000134}
135
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700136int32_t VCMGenericEncoder::Encode(const VideoFrame& inputFrame,
137 const CodecSpecificInfo* codecSpecificInfo,
138 const std::vector<FrameType>& frameTypes) {
pbosd9eec762015-11-17 06:03:43 -0800139 TRACE_EVENT1("webrtc", "VCMGenericEncoder::Encode", "timestamp",
140 inputFrame.timestamp());
141
pbos22993e12015-10-19 02:39:06 -0700142 for (FrameType frame_type : frameTypes)
143 RTC_DCHECK(frame_type == kVideoFrameKey || frame_type == kVideoFrameDelta);
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000144
145 rotation_ = inputFrame.rotation();
146
Peter Boström4f5db112015-10-29 16:53:59 +0100147 // Keep track of the current frame rotation and apply to the output of the
148 // encoder. There might not be exact as the encoder could have one frame delay
149 // but it should be close enough.
150 // TODO(pbos): Map from timestamp, this is racy (even if rotation_ is locked
151 // properly, which it isn't). More than one frame may be in the pipeline.
152 vcm_encoded_frame_callback_->SetRotation(rotation_);
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000153
pbos22993e12015-10-19 02:39:06 -0700154 int32_t result = encoder_->Encode(inputFrame, codecSpecificInfo, &frameTypes);
Peter Boströmb7d9a972015-12-18 16:01:11 +0100155
156 if (vcm_encoded_frame_callback_) {
157 vcm_encoded_frame_callback_->SignalLastEncoderImplementationUsed(
158 encoder_->ImplementationName());
159 }
160
Erik Språng2c4c9142015-06-24 11:24:44 +0200161 if (is_screenshare_ &&
162 result == WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT) {
163 // Target bitrate exceeded, encoder state has been reset - try again.
pbos22993e12015-10-19 02:39:06 -0700164 return encoder_->Encode(inputFrame, codecSpecificInfo, &frameTypes);
Erik Språng2c4c9142015-06-24 11:24:44 +0200165 }
166
167 return result;
niklase@google.com470e71d2011-07-07 08:21:25 +0000168}
169
Peter Boström69ccb332015-10-29 16:30:23 +0100170void VCMGenericEncoder::SetEncoderParameters(const EncoderParameters& params) {
171 bool channel_parameters_have_changed;
172 bool rates_have_changed;
173 {
174 rtc::CritScope lock(&params_lock_);
175 channel_parameters_have_changed =
176 params.loss_rate != encoder_params_.loss_rate ||
177 params.rtt != encoder_params_.rtt;
178 rates_have_changed =
179 params.target_bitrate != encoder_params_.target_bitrate ||
180 params.input_frame_rate != encoder_params_.input_frame_rate;
181 encoder_params_ = params;
182 }
183 if (channel_parameters_have_changed)
184 encoder_->SetChannelParameters(params.loss_rate, params.rtt);
185 if (rates_have_changed) {
186 uint32_t target_bitrate_kbps = (params.target_bitrate + 500) / 1000;
187 encoder_->SetRates(target_bitrate_kbps, params.input_frame_rate);
188 if (rate_observer_ != nullptr) {
189 rate_observer_->OnSetRates(params.target_bitrate,
190 params.input_frame_rate);
niklase@google.com470e71d2011-07-07 08:21:25 +0000191 }
Peter Boström69ccb332015-10-29 16:30:23 +0100192 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000193}
194
Peter Boström69ccb332015-10-29 16:30:23 +0100195EncoderParameters VCMGenericEncoder::GetEncoderParameters() const {
196 rtc::CritScope lock(&params_lock_);
197 return encoder_params_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000198}
199
philipel9d3ab612015-12-21 04:12:39 -0800200int32_t VCMGenericEncoder::SetPeriodicKeyFrames(bool enable) {
201 return encoder_->SetPeriodicKeyFrames(enable);
niklase@google.com470e71d2011-07-07 08:21:25 +0000202}
203
pbos@webrtc.org7b859cc2013-04-02 15:54:38 +0000204int32_t VCMGenericEncoder::RequestFrame(
stefan@webrtc.orgcf216862012-10-25 11:29:51 +0000205 const std::vector<FrameType>& frame_types) {
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700206 VideoFrame image;
pbos22993e12015-10-19 02:39:06 -0700207 return encoder_->Encode(image, NULL, &frame_types);
niklase@google.com470e71d2011-07-07 08:21:25 +0000208}
209
philipel9d3ab612015-12-21 04:12:39 -0800210bool VCMGenericEncoder::InternalSource() const {
211 return internal_source_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000212}
213
jackychen61b4d512015-04-21 15:30:11 -0700214void VCMGenericEncoder::OnDroppedFrame() {
215 encoder_->OnDroppedFrame();
216}
217
Peter Boströmeb66e802015-06-05 11:08:03 +0200218bool VCMGenericEncoder::SupportsNativeHandle() const {
219 return encoder_->SupportsNativeHandle();
220}
221
jackychen6e2ce6e2015-07-13 16:26:33 -0700222int VCMGenericEncoder::GetTargetFramerate() {
223 return encoder_->GetTargetFramerate();
224}
225
philipel9d3ab612015-12-21 04:12:39 -0800226/***************************
227 * Callback Implementation
228 ***************************/
andresp@webrtc.org1df9dc32014-01-09 08:01:57 +0000229VCMEncodedFrameCallback::VCMEncodedFrameCallback(
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000230 EncodedImageCallback* post_encode_callback)
Peter Boströmb7d9a972015-12-18 16:01:11 +0100231 : send_callback_(),
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000232 _mediaOpt(NULL),
233 _payloadType(0),
234 _internalSource(false),
235 _rotation(kVideoRotation_0),
236 post_encode_callback_(post_encode_callback)
tommi@webrtc.orga9da4c52012-07-20 11:17:23 +0000237#ifdef DEBUG_ENCODER_BIT_STREAM
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000238 ,
239 _bitStreamAfterEncoder(NULL)
tommi@webrtc.orga9da4c52012-07-20 11:17:23 +0000240#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000241{
242#ifdef DEBUG_ENCODER_BIT_STREAM
philipel9d3ab612015-12-21 04:12:39 -0800243 _bitStreamAfterEncoder = fopen("encoderBitStream.bit", "wb");
niklase@google.com470e71d2011-07-07 08:21:25 +0000244#endif
245}
246
philipel9d3ab612015-12-21 04:12:39 -0800247VCMEncodedFrameCallback::~VCMEncodedFrameCallback() {
niklase@google.com470e71d2011-07-07 08:21:25 +0000248#ifdef DEBUG_ENCODER_BIT_STREAM
philipel9d3ab612015-12-21 04:12:39 -0800249 fclose(_bitStreamAfterEncoder);
niklase@google.com470e71d2011-07-07 08:21:25 +0000250#endif
251}
252
philipel9d3ab612015-12-21 04:12:39 -0800253int32_t VCMEncodedFrameCallback::SetTransportCallback(
254 VCMPacketizationCallback* transport) {
255 send_callback_ = transport;
256 return VCM_OK;
niklase@google.com470e71d2011-07-07 08:21:25 +0000257}
258
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000259int32_t VCMEncodedFrameCallback::Encoded(
Peter Boströmb7d9a972015-12-18 16:01:11 +0100260 const EncodedImage& encoded_image,
niklase@google.com470e71d2011-07-07 08:21:25 +0000261 const CodecSpecificInfo* codecSpecificInfo,
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000262 const RTPFragmentationHeader* fragmentationHeader) {
pbosd9eec762015-11-17 06:03:43 -0800263 TRACE_EVENT_INSTANT1("webrtc", "VCMEncodedFrameCallback::Encoded",
Peter Boströmb7d9a972015-12-18 16:01:11 +0100264 "timestamp", encoded_image._timeStamp);
265 post_encode_callback_->Encoded(encoded_image, NULL, NULL);
andresp@webrtc.org1df9dc32014-01-09 08:01:57 +0000266
Peter Boströmb7d9a972015-12-18 16:01:11 +0100267 if (send_callback_ == NULL) {
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000268 return VCM_UNINITIALIZED;
269 }
270
wjia@webrtc.orgf0cd3942011-10-25 00:40:43 +0000271#ifdef DEBUG_ENCODER_BIT_STREAM
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000272 if (_bitStreamAfterEncoder != NULL) {
Peter Boströmb7d9a972015-12-18 16:01:11 +0100273 fwrite(encoded_image._buffer, 1, encoded_image._length,
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000274 _bitStreamAfterEncoder);
275 }
wjia@webrtc.orgf0cd3942011-10-25 00:40:43 +0000276#endif
niklase@google.com470e71d2011-07-07 08:21:25 +0000277
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000278 RTPVideoHeader rtpVideoHeader;
glaznev@webrtc.org30540fe2015-02-18 20:30:03 +0000279 memset(&rtpVideoHeader, 0, sizeof(RTPVideoHeader));
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000280 RTPVideoHeader* rtpVideoHeaderPtr = &rtpVideoHeader;
Guo-wei Shieh2c370782015-04-08 13:00:10 -0700281 if (codecSpecificInfo) {
282 CopyCodecSpecific(codecSpecificInfo, rtpVideoHeaderPtr);
283 }
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +0000284 rtpVideoHeader.rotation = _rotation;
niklase@google.com470e71d2011-07-07 08:21:25 +0000285
Peter Boströmb7d9a972015-12-18 16:01:11 +0100286 int32_t callbackReturn = send_callback_->SendData(
287 _payloadType, encoded_image, *fragmentationHeader, rtpVideoHeaderPtr);
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000288 if (callbackReturn < 0) {
289 return callbackReturn;
290 }
291
292 if (_mediaOpt != NULL) {
Peter Boströmb7d9a972015-12-18 16:01:11 +0100293 _mediaOpt->UpdateWithEncodedData(encoded_image);
changbin.shao@webrtc.orgf31f56d2015-02-09 09:14:03 +0000294 if (_internalSource)
295 return _mediaOpt->DropFrame(); // Signal to encoder to drop next frame.
296 }
297 return VCM_OK;
niklase@google.com470e71d2011-07-07 08:21:25 +0000298}
299
Peter Boströmb7d9a972015-12-18 16:01:11 +0100300void VCMEncodedFrameCallback::SetMediaOpt(
301 media_optimization::MediaOptimization* mediaOpt) {
302 _mediaOpt = mediaOpt;
303}
304
305void VCMEncodedFrameCallback::SignalLastEncoderImplementationUsed(
306 const char* implementation_name) {
307 if (send_callback_)
308 send_callback_->OnEncoderImplementationName(implementation_name);
niklase@google.com470e71d2011-07-07 08:21:25 +0000309}
310
pbos@webrtc.orgd900e8b2013-07-03 15:12:26 +0000311} // namespace webrtc