blob: 7e2a678a563f7883c5bab7415997ca90fb18935f [file] [log] [blame]
magjed614d5b72016-11-15 06:30:54 -08001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "webrtc/media/engine/videoencodersoftwarefallbackwrapper.h"
12
emircan82fac892017-08-23 14:19:50 -070013#include "webrtc/media/base/h264_profile_level_id.h"
magjed509e4fe2016-11-18 01:34:11 -080014#include "webrtc/media/engine/internalencoderfactory.h"
magjed614d5b72016-11-15 06:30:54 -080015#include "webrtc/modules/video_coding/include/video_error_codes.h"
asapersson22c76c42017-08-16 00:53:59 -070016#include "webrtc/rtc_base/checks.h"
Edward Lemurc20978e2017-07-06 19:44:34 +020017#include "webrtc/rtc_base/logging.h"
asapersson22c76c42017-08-16 00:53:59 -070018#include "webrtc/rtc_base/timeutils.h"
19#include "webrtc/system_wrappers/include/field_trial.h"
magjed614d5b72016-11-15 06:30:54 -080020
21namespace webrtc {
asapersson22c76c42017-08-16 00:53:59 -070022namespace {
23const char kVp8ForceFallbackEncoderFieldTrial[] =
24 "WebRTC-VP8-Forced-Fallback-Encoder";
25
26bool EnableForcedFallback(const cricket::VideoCodec& codec) {
27 if (!webrtc::field_trial::IsEnabled(kVp8ForceFallbackEncoderFieldTrial))
28 return false;
29
kthelgason1cdddc92017-08-24 03:52:48 -070030 return (PayloadStringToCodecType(codec.name) == kVideoCodecVP8);
asapersson22c76c42017-08-16 00:53:59 -070031}
32
33bool IsForcedFallbackPossible(const VideoCodec& codec_settings) {
34 return codec_settings.codecType == kVideoCodecVP8 &&
35 codec_settings.numberOfSimulcastStreams <= 1 &&
36 codec_settings.VP8().numberOfTemporalLayers == 1;
37}
38
39void GetForcedFallbackParamsFromFieldTrialGroup(uint32_t* param_low_kbps,
40 uint32_t* param_high_kbps,
41 int64_t* param_min_low_ms) {
42 RTC_DCHECK(param_low_kbps);
43 RTC_DCHECK(param_high_kbps);
44 RTC_DCHECK(param_min_low_ms);
45 std::string group =
46 webrtc::field_trial::FindFullName(kVp8ForceFallbackEncoderFieldTrial);
47 if (group.empty())
48 return;
49
50 int low_kbps;
51 int high_kbps;
52 int min_low_ms;
asapersson142fcc92017-08-17 08:58:54 -070053 int min_pixels;
54 if (sscanf(group.c_str(), "Enabled-%d,%d,%d,%d", &low_kbps, &high_kbps,
55 &min_low_ms, &min_pixels) != 4) {
asapersson22c76c42017-08-16 00:53:59 -070056 LOG(LS_WARNING) << "Invalid number of forced fallback parameters provided.";
57 return;
58 }
asapersson142fcc92017-08-17 08:58:54 -070059 if (min_low_ms <= 0 || min_pixels <= 0 || low_kbps <= 0 ||
60 high_kbps <= low_kbps) {
asapersson22c76c42017-08-16 00:53:59 -070061 LOG(LS_WARNING) << "Invalid forced fallback parameter value provided.";
62 return;
63 }
64 *param_low_kbps = low_kbps;
65 *param_high_kbps = high_kbps;
66 *param_min_low_ms = min_low_ms;
67}
68} // namespace
magjed614d5b72016-11-15 06:30:54 -080069
70VideoEncoderSoftwareFallbackWrapper::VideoEncoderSoftwareFallbackWrapper(
magjed509e4fe2016-11-18 01:34:11 -080071 const cricket::VideoCodec& codec,
magjedf52d34d2017-08-29 00:58:52 -070072 std::unique_ptr<webrtc::VideoEncoder> encoder)
Erik Språng08127a92016-11-16 16:41:30 +010073 : number_of_cores_(0),
74 max_payload_size_(0),
75 rates_set_(false),
76 framerate_(0),
magjed614d5b72016-11-15 06:30:54 -080077 channel_parameters_set_(false),
Erik Språng08127a92016-11-16 16:41:30 +010078 packet_loss_(0),
79 rtt_(0),
magjed509e4fe2016-11-18 01:34:11 -080080 codec_(codec),
magjedf52d34d2017-08-29 00:58:52 -070081 encoder_(std::move(encoder)),
asapersson22c76c42017-08-16 00:53:59 -070082 callback_(nullptr),
83 forced_fallback_possible_(EnableForcedFallback(codec)) {
84 if (forced_fallback_possible_) {
85 GetForcedFallbackParamsFromFieldTrialGroup(&forced_fallback_.low_kbps,
86 &forced_fallback_.high_kbps,
87 &forced_fallback_.min_low_ms);
88 }
89}
magjed614d5b72016-11-15 06:30:54 -080090
91bool VideoEncoderSoftwareFallbackWrapper::InitFallbackEncoder() {
brandtr955d7f12017-08-24 05:19:57 -070092 LOG(LS_WARNING) << "Encoder falling back to software encoding.";
emircan82fac892017-08-23 14:19:50 -070093 MaybeModifyCodecForFallback();
magjed509e4fe2016-11-18 01:34:11 -080094 cricket::InternalEncoderFactory internal_factory;
95 if (!FindMatchingCodec(internal_factory.supported_codecs(), codec_)) {
magjed614d5b72016-11-15 06:30:54 -080096 LOG(LS_WARNING)
97 << "Encoder requesting fallback to codec not supported in software.";
98 return false;
99 }
magjed509e4fe2016-11-18 01:34:11 -0800100 fallback_encoder_.reset(internal_factory.CreateVideoEncoder(codec_));
magjed614d5b72016-11-15 06:30:54 -0800101 if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_,
102 max_payload_size_) !=
103 WEBRTC_VIDEO_CODEC_OK) {
104 LOG(LS_ERROR) << "Failed to initialize software-encoder fallback.";
105 fallback_encoder_->Release();
106 fallback_encoder_.reset();
107 return false;
108 }
109 // Replay callback, rates, and channel parameters.
110 if (callback_)
111 fallback_encoder_->RegisterEncodeCompleteCallback(callback_);
112 if (rates_set_)
Erik Språng08127a92016-11-16 16:41:30 +0100113 fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate_);
magjed614d5b72016-11-15 06:30:54 -0800114 if (channel_parameters_set_)
115 fallback_encoder_->SetChannelParameters(packet_loss_, rtt_);
116
117 fallback_implementation_name_ =
118 std::string(fallback_encoder_->ImplementationName()) +
119 " (fallback from: " + encoder_->ImplementationName() + ")";
120 // Since we're switching to the fallback encoder, Release the real encoder. It
121 // may be re-initialized via InitEncode later, and it will continue to get
122 // Set calls for rates and channel parameters in the meantime.
123 encoder_->Release();
124 return true;
125}
126
127int32_t VideoEncoderSoftwareFallbackWrapper::InitEncode(
128 const VideoCodec* codec_settings,
129 int32_t number_of_cores,
130 size_t max_payload_size) {
131 // Store settings, in case we need to dynamically switch to the fallback
132 // encoder after a failed Encode call.
133 codec_settings_ = *codec_settings;
134 number_of_cores_ = number_of_cores;
135 max_payload_size_ = max_payload_size;
136 // Clear stored rate/channel parameters.
137 rates_set_ = false;
138 channel_parameters_set_ = false;
asapersson22c76c42017-08-16 00:53:59 -0700139 ValidateSettingsForForcedFallback();
140
141 // Try to reinit forced software codec if it is in use.
142 if (TryReInitForcedFallbackEncoder()) {
143 return WEBRTC_VIDEO_CODEC_OK;
144 }
145 forced_fallback_.Reset();
magjed614d5b72016-11-15 06:30:54 -0800146
147 int32_t ret =
148 encoder_->InitEncode(codec_settings, number_of_cores, max_payload_size);
magjed509e4fe2016-11-18 01:34:11 -0800149 if (ret == WEBRTC_VIDEO_CODEC_OK || codec_.name.empty()) {
brandtr955d7f12017-08-24 05:19:57 -0700150 if (fallback_encoder_) {
151 LOG(LS_WARNING)
152 << "InitEncode OK, no longer using the software fallback encoder.";
magjed614d5b72016-11-15 06:30:54 -0800153 fallback_encoder_->Release();
brandtr955d7f12017-08-24 05:19:57 -0700154 }
magjed614d5b72016-11-15 06:30:54 -0800155 fallback_encoder_.reset();
156 if (callback_)
157 encoder_->RegisterEncodeCompleteCallback(callback_);
158 return ret;
159 }
160 // Try to instantiate software codec.
161 if (InitFallbackEncoder()) {
162 return WEBRTC_VIDEO_CODEC_OK;
163 }
164 // Software encoder failed, use original return code.
165 return ret;
166}
167
168int32_t VideoEncoderSoftwareFallbackWrapper::RegisterEncodeCompleteCallback(
169 EncodedImageCallback* callback) {
170 callback_ = callback;
171 int32_t ret = encoder_->RegisterEncodeCompleteCallback(callback);
172 if (fallback_encoder_)
173 return fallback_encoder_->RegisterEncodeCompleteCallback(callback);
174 return ret;
175}
176
177int32_t VideoEncoderSoftwareFallbackWrapper::Release() {
178 // If the fallback_encoder_ is non-null, it means it was created via
179 // InitFallbackEncoder which has Release()d encoder_, so we should only ever
180 // need to Release() whichever one is active.
181 if (fallback_encoder_)
182 return fallback_encoder_->Release();
183 return encoder_->Release();
184}
185
186int32_t VideoEncoderSoftwareFallbackWrapper::Encode(
187 const VideoFrame& frame,
188 const CodecSpecificInfo* codec_specific_info,
189 const std::vector<FrameType>* frame_types) {
asapersson22c76c42017-08-16 00:53:59 -0700190 if (TryReleaseForcedFallbackEncoder()) {
191 // Frame may have been converted from kNative to kI420 during fallback.
192 if (encoder_->SupportsNativeHandle() &&
193 frame.video_frame_buffer()->type() != VideoFrameBuffer::Type::kNative) {
194 LOG(LS_WARNING) << "Encoder supports native frames, dropping one frame "
195 << "to avoid possible reconfig due to format change.";
196 return WEBRTC_VIDEO_CODEC_ERROR;
197 }
198 }
magjed614d5b72016-11-15 06:30:54 -0800199 if (fallback_encoder_)
200 return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
201 int32_t ret = encoder_->Encode(frame, codec_specific_info, frame_types);
202 // If requested, try a software fallback.
asapersson22c76c42017-08-16 00:53:59 -0700203 bool fallback_requested =
204 (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE) ||
205 (ret == WEBRTC_VIDEO_CODEC_OK && RequestForcedFallback());
206 if (fallback_requested && InitFallbackEncoder()) {
207 // Fallback was successful.
208 if (ret == WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE)
209 forced_fallback_.Reset(); // Not a forced fallback.
Magnus Jedvert7a721e82017-06-14 11:28:08 +0200210 if (frame.video_frame_buffer()->type() == VideoFrameBuffer::Type::kNative &&
magjed614d5b72016-11-15 06:30:54 -0800211 !fallback_encoder_->SupportsNativeHandle()) {
212 LOG(LS_WARNING) << "Fallback encoder doesn't support native frames, "
213 << "dropping one frame.";
214 return WEBRTC_VIDEO_CODEC_ERROR;
215 }
216
asapersson22c76c42017-08-16 00:53:59 -0700217 // Start using the fallback with this frame.
magjed614d5b72016-11-15 06:30:54 -0800218 return fallback_encoder_->Encode(frame, codec_specific_info, frame_types);
219 }
220 return ret;
221}
222
223int32_t VideoEncoderSoftwareFallbackWrapper::SetChannelParameters(
224 uint32_t packet_loss,
225 int64_t rtt) {
226 channel_parameters_set_ = true;
227 packet_loss_ = packet_loss;
228 rtt_ = rtt;
229 int32_t ret = encoder_->SetChannelParameters(packet_loss, rtt);
230 if (fallback_encoder_)
231 return fallback_encoder_->SetChannelParameters(packet_loss, rtt);
232 return ret;
233}
234
Erik Språng08127a92016-11-16 16:41:30 +0100235int32_t VideoEncoderSoftwareFallbackWrapper::SetRateAllocation(
236 const BitrateAllocation& bitrate_allocation,
237 uint32_t framerate) {
magjed614d5b72016-11-15 06:30:54 -0800238 rates_set_ = true;
Erik Språng08127a92016-11-16 16:41:30 +0100239 bitrate_allocation_ = bitrate_allocation;
magjed614d5b72016-11-15 06:30:54 -0800240 framerate_ = framerate;
Erik Språng08127a92016-11-16 16:41:30 +0100241 int32_t ret = encoder_->SetRateAllocation(bitrate_allocation_, framerate);
magjed614d5b72016-11-15 06:30:54 -0800242 if (fallback_encoder_)
Erik Språng08127a92016-11-16 16:41:30 +0100243 return fallback_encoder_->SetRateAllocation(bitrate_allocation_, framerate);
magjed614d5b72016-11-15 06:30:54 -0800244 return ret;
245}
246
magjed614d5b72016-11-15 06:30:54 -0800247bool VideoEncoderSoftwareFallbackWrapper::SupportsNativeHandle() const {
248 if (fallback_encoder_)
249 return fallback_encoder_->SupportsNativeHandle();
250 return encoder_->SupportsNativeHandle();
251}
252
kthelgason876222f2016-11-29 01:44:11 -0800253VideoEncoder::ScalingSettings
254VideoEncoderSoftwareFallbackWrapper::GetScalingSettings() const {
asapersson142fcc92017-08-17 08:58:54 -0700255 if (forced_fallback_possible_ && fallback_encoder_)
256 return fallback_encoder_->GetScalingSettings();
kthelgason876222f2016-11-29 01:44:11 -0800257 return encoder_->GetScalingSettings();
258}
259
kthelgason535dbd32017-01-26 00:36:31 -0800260const char *VideoEncoderSoftwareFallbackWrapper::ImplementationName() const {
261 if (fallback_encoder_)
262 return fallback_encoder_->ImplementationName();
263 return encoder_->ImplementationName();
264}
265
asapersson22c76c42017-08-16 00:53:59 -0700266bool VideoEncoderSoftwareFallbackWrapper::IsForcedFallbackActive() const {
267 return (forced_fallback_possible_ && fallback_encoder_ &&
268 forced_fallback_.start_ms);
269}
270
271bool VideoEncoderSoftwareFallbackWrapper::RequestForcedFallback() {
272 if (!forced_fallback_possible_ || fallback_encoder_ || !rates_set_)
273 return false;
274
275 // No fallback encoder.
276 return forced_fallback_.ShouldStart(bitrate_allocation_.get_sum_kbps(),
277 codec_settings_);
278}
279
280bool VideoEncoderSoftwareFallbackWrapper::TryReleaseForcedFallbackEncoder() {
281 if (!IsForcedFallbackActive())
282 return false;
283
asapersson142fcc92017-08-17 08:58:54 -0700284 if (!forced_fallback_.ShouldStop(bitrate_allocation_.get_sum_kbps(),
285 codec_settings_)) {
asapersson22c76c42017-08-16 00:53:59 -0700286 return false;
asapersson142fcc92017-08-17 08:58:54 -0700287 }
asapersson22c76c42017-08-16 00:53:59 -0700288
289 // Release the forced fallback encoder.
290 if (encoder_->InitEncode(&codec_settings_, number_of_cores_,
291 max_payload_size_) == WEBRTC_VIDEO_CODEC_OK) {
292 LOG(LS_INFO) << "Stop forced SW encoder fallback, max bitrate exceeded.";
293 fallback_encoder_->Release();
294 fallback_encoder_.reset();
295 forced_fallback_.Reset();
296 return true;
297 }
298 return false;
299}
300
301bool VideoEncoderSoftwareFallbackWrapper::TryReInitForcedFallbackEncoder() {
302 if (!IsForcedFallbackActive())
303 return false;
304
305 // Encoder reconfigured.
306 if (!forced_fallback_.IsValid(codec_settings_)) {
307 LOG(LS_INFO) << "Stop forced SW encoder fallback, max pixels exceeded.";
308 return false;
309 }
310 // Settings valid, reinitialize the forced fallback encoder.
311 if (fallback_encoder_->InitEncode(&codec_settings_, number_of_cores_,
312 max_payload_size_) !=
313 WEBRTC_VIDEO_CODEC_OK) {
314 LOG(LS_ERROR) << "Failed to init forced SW encoder fallback.";
315 return false;
316 }
317 return true;
318}
319
320void VideoEncoderSoftwareFallbackWrapper::ValidateSettingsForForcedFallback() {
321 if (!forced_fallback_possible_)
322 return;
323
324 if (!IsForcedFallbackPossible(codec_settings_)) {
325 if (IsForcedFallbackActive()) {
326 fallback_encoder_->Release();
327 fallback_encoder_.reset();
328 }
329 LOG(LS_INFO) << "Disable forced_fallback_possible_ due to settings.";
330 forced_fallback_possible_ = false;
331 }
332}
333
334bool VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams::ShouldStart(
335 uint32_t bitrate_kbps,
336 const VideoCodec& codec) {
337 if (bitrate_kbps > low_kbps || !IsValid(codec)) {
338 start_ms.reset();
339 return false;
340 }
341
342 // Has bitrate been below |low_kbps| for long enough duration.
343 int64_t now_ms = rtc::TimeMillis();
344 if (!start_ms)
345 start_ms.emplace(now_ms);
346
347 if ((now_ms - *start_ms) >= min_low_ms) {
348 LOG(LS_INFO) << "Request forced SW encoder fallback.";
349 // In case the request fails, update time to avoid too frequent requests.
350 start_ms.emplace(now_ms);
351 return true;
352 }
353 return false;
354}
355
356bool VideoEncoderSoftwareFallbackWrapper::ForcedFallbackParams::ShouldStop(
asapersson142fcc92017-08-17 08:58:54 -0700357 uint32_t bitrate_kbps,
358 const VideoCodec& codec) const {
359 return bitrate_kbps >= high_kbps &&
360 (codec.width * codec.height >= kMinPixelsStop);
asapersson22c76c42017-08-16 00:53:59 -0700361}
362
emircan82fac892017-08-23 14:19:50 -0700363void VideoEncoderSoftwareFallbackWrapper::MaybeModifyCodecForFallback() {
364 // We have a specific case for H264 ConstrainedBaseline because that is the
365 // only supported profile in Sw fallback.
366 if (!cricket::CodecNamesEq(codec_.name.c_str(), cricket::kH264CodecName))
367 return;
368 codec_.SetParam(cricket::kH264FmtpProfileLevelId,
369 cricket::kH264ProfileLevelConstrainedBaseline);
370}
371
magjed614d5b72016-11-15 06:30:54 -0800372} // namespace webrtc