blob: 3fc56b5095ce5f9bc1358375fdd739e5c99b9f9b [file] [log] [blame]
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00001/*
kjellander1afca732016-02-07 20:46:45 -08002 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00003 *
kjellander1afca732016-02-07 20:46:45 -08004 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00009 */
10
henrike@webrtc.org28e20752013-07-10 00:45:36 +000011// Implementation file of class VideoCapturer.
12
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020013#include "media/base/videocapturer.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000014
15#include <algorithm>
16
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "api/video/i420_buffer.h"
18#include "api/video/video_frame.h"
19#include "rtc_base/logging.h"
Ilya Nikolaevskiya0b66c72017-10-24 13:51:26 +020020#include "system_wrappers/include/field_trial.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000021
henrike@webrtc.org28e20752013-07-10 00:45:36 +000022namespace cricket {
23
24namespace {
25
Peter Boström0c4e06b2015-10-07 12:23:21 +020026static const int64_t kMaxDistance = ~(static_cast<int64_t>(1) << 63);
kjellanderfcfc8042016-01-14 11:01:09 -080027#ifdef WEBRTC_LINUX
henrike@webrtc.org28e20752013-07-10 00:45:36 +000028static const int kYU12Penalty = 16; // Needs to be higher than MJPG index.
henrike@webrtc.orgf5bebd42014-04-04 18:39:07 +000029#endif
Ilya Nikolaevskiya0b66c72017-10-24 13:51:26 +020030static const char* kSimulcastScreenshareFieldTrialName =
31 "WebRTC-SimulcastScreenshare";
henrike@webrtc.org28e20752013-07-10 00:45:36 +000032
33} // namespace
34
35/////////////////////////////////////////////////////////////////////
henrike@webrtc.org28e20752013-07-10 00:45:36 +000036// Implementation of class VideoCapturer
37/////////////////////////////////////////////////////////////////////
deadbeeff5629ad2016-03-18 11:38:26 -070038VideoCapturer::VideoCapturer() : apply_rotation_(false) {
Perfb45d172016-02-29 12:07:35 +010039 thread_checker_.DetachFromThread();
henrike@webrtc.org28e20752013-07-10 00:45:36 +000040 Construct();
41}
42
43void VideoCapturer::Construct() {
henrike@webrtc.org28e20752013-07-10 00:45:36 +000044 enable_camera_list_ = false;
45 capture_state_ = CS_STOPPED;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000046 scaled_width_ = 0;
47 scaled_height_ = 0;
henrike@webrtc.orga7b98182014-02-21 15:51:43 +000048 enable_video_adapter_ = true;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000049}
50
51const std::vector<VideoFormat>* VideoCapturer::GetSupportedFormats() const {
52 return &filtered_supported_formats_;
53}
54
55bool VideoCapturer::StartCapturing(const VideoFormat& capture_format) {
Perfb45d172016-02-29 12:07:35 +010056 RTC_DCHECK(thread_checker_.CalledOnValidThread());
henrike@webrtc.org28e20752013-07-10 00:45:36 +000057 CaptureState result = Start(capture_format);
58 const bool success = (result == CS_RUNNING) || (result == CS_STARTING);
59 if (!success) {
60 return false;
61 }
62 if (result == CS_RUNNING) {
63 SetCaptureState(result);
64 }
65 return true;
66}
67
henrike@webrtc.org28e20752013-07-10 00:45:36 +000068void VideoCapturer::SetSupportedFormats(
69 const std::vector<VideoFormat>& formats) {
Perfb45d172016-02-29 12:07:35 +010070 // This method is OK to call during initialization on a separate thread.
71 RTC_DCHECK(capture_state_ == CS_STOPPED ||
72 thread_checker_.CalledOnValidThread());
henrike@webrtc.org28e20752013-07-10 00:45:36 +000073 supported_formats_ = formats;
74 UpdateFilteredSupportedFormats();
75}
76
77bool VideoCapturer::GetBestCaptureFormat(const VideoFormat& format,
78 VideoFormat* best_format) {
Perfb45d172016-02-29 12:07:35 +010079 RTC_DCHECK(thread_checker_.CalledOnValidThread());
henrike@webrtc.org28e20752013-07-10 00:45:36 +000080 // TODO(fbarchard): Directly support max_format.
81 UpdateFilteredSupportedFormats();
82 const std::vector<VideoFormat>* supported_formats = GetSupportedFormats();
83
84 if (supported_formats->empty()) {
85 return false;
86 }
87 LOG(LS_INFO) << " Capture Requested " << format.ToString();
Peter Boström0c4e06b2015-10-07 12:23:21 +020088 int64_t best_distance = kMaxDistance;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000089 std::vector<VideoFormat>::const_iterator best = supported_formats->end();
90 std::vector<VideoFormat>::const_iterator i;
91 for (i = supported_formats->begin(); i != supported_formats->end(); ++i) {
Peter Boström0c4e06b2015-10-07 12:23:21 +020092 int64_t distance = GetFormatDistance(format, *i);
henrike@webrtc.org28e20752013-07-10 00:45:36 +000093 // TODO(fbarchard): Reduce to LS_VERBOSE if/when camera capture is
94 // relatively bug free.
95 LOG(LS_INFO) << " Supported " << i->ToString() << " distance " << distance;
96 if (distance < best_distance) {
97 best_distance = distance;
98 best = i;
99 }
100 }
101 if (supported_formats->end() == best) {
102 LOG(LS_ERROR) << " No acceptable camera format found";
103 return false;
104 }
105
106 if (best_format) {
107 best_format->width = best->width;
108 best_format->height = best->height;
109 best_format->fourcc = best->fourcc;
sergeyu@chromium.org4b26e2e2014-01-15 23:15:54 +0000110 best_format->interval = best->interval;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000111 LOG(LS_INFO) << " Best " << best_format->ToString() << " Interval "
112 << best_format->interval << " distance " << best_distance;
113 }
114 return true;
115}
116
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000117void VideoCapturer::ConstrainSupportedFormats(const VideoFormat& max_format) {
Perfb45d172016-02-29 12:07:35 +0100118 RTC_DCHECK(thread_checker_.CalledOnValidThread());
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000119 max_format_.reset(new VideoFormat(max_format));
120 LOG(LS_VERBOSE) << " ConstrainSupportedFormats " << max_format.ToString();
121 UpdateFilteredSupportedFormats();
122}
123
nissefcc640f2016-04-01 01:10:42 -0700124bool VideoCapturer::GetInputSize(int* width, int* height) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000125 rtc::CritScope cs(&frame_stats_crit_);
nissefcc640f2016-04-01 01:10:42 -0700126 if (!input_size_valid_) {
127 return false;
128 }
129 *width = input_width_;
130 *height = input_height_;
131
132 return true;
henrike@webrtc.org704bf9e2014-02-27 17:52:04 +0000133}
134
Pera5092412016-02-12 13:30:57 +0100135void VideoCapturer::RemoveSink(
nisseacd935b2016-11-11 03:55:13 -0800136 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink) {
Perfb45d172016-02-29 12:07:35 +0100137 RTC_DCHECK(thread_checker_.CalledOnValidThread());
Pera5092412016-02-12 13:30:57 +0100138 broadcaster_.RemoveSink(sink);
perkj2d5f0912016-02-29 00:04:41 -0800139 OnSinkWantsChanged(broadcaster_.wants());
Pera5092412016-02-12 13:30:57 +0100140}
141
142void VideoCapturer::AddOrUpdateSink(
nisseacd935b2016-11-11 03:55:13 -0800143 rtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
Pera5092412016-02-12 13:30:57 +0100144 const rtc::VideoSinkWants& wants) {
Perfb45d172016-02-29 12:07:35 +0100145 RTC_DCHECK(thread_checker_.CalledOnValidThread());
Pera5092412016-02-12 13:30:57 +0100146 broadcaster_.AddOrUpdateSink(sink, wants);
147 OnSinkWantsChanged(broadcaster_.wants());
148}
149
150void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) {
Perfb45d172016-02-29 12:07:35 +0100151 RTC_DCHECK(thread_checker_.CalledOnValidThread());
Pera5092412016-02-12 13:30:57 +0100152 apply_rotation_ = wants.rotation_applied;
perkj2d5f0912016-02-29 00:04:41 -0800153
154 if (video_adapter()) {
sprangc5d62e22017-04-02 23:53:04 -0700155 video_adapter()->OnResolutionFramerateRequest(wants.target_pixel_count,
156 wants.max_pixel_count,
157 wants.max_framerate_fps);
perkj2d5f0912016-02-29 00:04:41 -0800158 }
Pera5092412016-02-12 13:30:57 +0100159}
160
nisse47ac4622016-05-25 08:47:01 -0700161bool VideoCapturer::AdaptFrame(int width,
162 int height,
nisse191b3592016-06-22 08:36:53 -0700163 int64_t camera_time_us,
164 int64_t system_time_us,
nisse47ac4622016-05-25 08:47:01 -0700165 int* out_width,
166 int* out_height,
167 int* crop_width,
168 int* crop_height,
169 int* crop_x,
nisse191b3592016-06-22 08:36:53 -0700170 int* crop_y,
171 int64_t* translated_camera_time_us) {
nissea0758482016-09-14 00:37:00 -0700172 if (translated_camera_time_us) {
173 *translated_camera_time_us =
174 timestamp_aligner_.TranslateTimestamp(camera_time_us, system_time_us);
175 }
Pera5092412016-02-12 13:30:57 +0100176 if (!broadcaster_.frame_wanted()) {
nisse47ac4622016-05-25 08:47:01 -0700177 return false;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000178 }
braveyao@webrtc.org086c8d52014-12-22 05:46:42 +0000179
Ilya Nikolaevskiya0b66c72017-10-24 13:51:26 +0200180 bool simulcast_screenshare_enabled =
181 webrtc::field_trial::IsEnabled(kSimulcastScreenshareFieldTrialName);
182 if (enable_video_adapter_ &&
183 (!IsScreencast() || simulcast_screenshare_enabled)) {
nisse47ac4622016-05-25 08:47:01 -0700184 if (!video_adapter_.AdaptFrameResolution(
nisse191b3592016-06-22 08:36:53 -0700185 width, height, camera_time_us * rtc::kNumNanosecsPerMicrosec,
nisse47ac4622016-05-25 08:47:01 -0700186 crop_width, crop_height, out_width, out_height)) {
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000187 // VideoAdapter dropped the frame.
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200188 broadcaster_.OnDiscardedFrame();
nisse47ac4622016-05-25 08:47:01 -0700189 return false;
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000190 }
nisse47ac4622016-05-25 08:47:01 -0700191 *crop_x = (width - *crop_width) / 2;
192 *crop_y = (height - *crop_height) / 2;
193 } else {
194 *out_width = width;
195 *out_height = height;
196 *crop_width = width;
197 *crop_height = height;
198 *crop_x = 0;
199 *crop_y = 0;
200 }
nisse191b3592016-06-22 08:36:53 -0700201
nisse47ac4622016-05-25 08:47:01 -0700202 return true;
203}
204
nisseacd935b2016-11-11 03:55:13 -0800205void VideoCapturer::OnFrame(const webrtc::VideoFrame& frame,
nisse47ac4622016-05-25 08:47:01 -0700206 int orig_width,
207 int orig_height) {
nissef5297a02016-09-30 01:34:27 -0700208 // For a child class which implements rotation itself, we should
209 // always have apply_rotation_ == false or frame.rotation() == 0.
210 // Except possibly during races where apply_rotation_ is changed
211 // mid-stream.
212 if (apply_rotation_ && frame.rotation() != webrtc::kVideoRotation_0) {
213 rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer(
214 frame.video_frame_buffer());
magjed3f075492017-06-01 10:02:26 -0700215 if (buffer->type() != webrtc::VideoFrameBuffer::Type::kI420) {
216 // Sources producing non-I420 frames must handle apply_rotation
nissef5297a02016-09-30 01:34:27 -0700217 // themselves. But even if they do, we may occasionally end up
218 // in this case, for frames in flight at the time
219 // applied_rotation is set to true. In that case, we just drop
220 // the frame.
magjed3f075492017-06-01 10:02:26 -0700221 LOG(LS_WARNING) << "Non-I420 frame requiring rotation. Discarding.";
nissef5297a02016-09-30 01:34:27 -0700222 return;
223 }
nisseacd935b2016-11-11 03:55:13 -0800224 broadcaster_.OnFrame(webrtc::VideoFrame(
magjed3f075492017-06-01 10:02:26 -0700225 webrtc::I420Buffer::Rotate(*buffer->GetI420(), frame.rotation()),
nissef5297a02016-09-30 01:34:27 -0700226 webrtc::kVideoRotation_0, frame.timestamp_us()));
227 } else {
228 broadcaster_.OnFrame(frame);
229 }
nisse47ac4622016-05-25 08:47:01 -0700230 UpdateInputSize(orig_width, orig_height);
Pera5092412016-02-12 13:30:57 +0100231}
232
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000233void VideoCapturer::SetCaptureState(CaptureState state) {
Perfb45d172016-02-29 12:07:35 +0100234 RTC_DCHECK(thread_checker_.CalledOnValidThread());
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000235 if (state == capture_state_) {
236 // Don't trigger a state changed callback if the state hasn't changed.
237 return;
238 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000239 capture_state_ = state;
Perfb45d172016-02-29 12:07:35 +0100240 SignalStateChange(this, capture_state_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000241}
242
243// Get the distance between the supported and desired formats.
244// Prioritization is done according to this algorithm:
245// 1) Width closeness. If not same, we prefer wider.
246// 2) Height closeness. If not same, we prefer higher.
247// 3) Framerate closeness. If not same, we prefer faster.
248// 4) Compression. If desired format has a specific fourcc, we need exact match;
249// otherwise, we use preference.
Peter Boström0c4e06b2015-10-07 12:23:21 +0200250int64_t VideoCapturer::GetFormatDistance(const VideoFormat& desired,
251 const VideoFormat& supported) {
Perfb45d172016-02-29 12:07:35 +0100252 RTC_DCHECK(thread_checker_.CalledOnValidThread());
Peter Boström0c4e06b2015-10-07 12:23:21 +0200253 int64_t distance = kMaxDistance;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000254
255 // Check fourcc.
Peter Boström0c4e06b2015-10-07 12:23:21 +0200256 uint32_t supported_fourcc = CanonicalFourCC(supported.fourcc);
257 int64_t delta_fourcc = kMaxDistance;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000258 if (FOURCC_ANY == desired.fourcc) {
259 // Any fourcc is OK for the desired. Use preference to find best fourcc.
Peter Boström0c4e06b2015-10-07 12:23:21 +0200260 std::vector<uint32_t> preferred_fourccs;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000261 if (!GetPreferredFourccs(&preferred_fourccs)) {
262 return distance;
263 }
264
265 for (size_t i = 0; i < preferred_fourccs.size(); ++i) {
266 if (supported_fourcc == CanonicalFourCC(preferred_fourccs[i])) {
267 delta_fourcc = i;
kjellanderfcfc8042016-01-14 11:01:09 -0800268#ifdef WEBRTC_LINUX
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000269 // For HD avoid YU12 which is a software conversion and has 2 bugs
270 // b/7326348 b/6960899. Reenable when fixed.
271 if (supported.height >= 720 && (supported_fourcc == FOURCC_YU12 ||
272 supported_fourcc == FOURCC_YV12)) {
273 delta_fourcc += kYU12Penalty;
274 }
275#endif
276 break;
277 }
278 }
279 } else if (supported_fourcc == CanonicalFourCC(desired.fourcc)) {
280 delta_fourcc = 0; // Need exact match.
281 }
282
283 if (kMaxDistance == delta_fourcc) {
284 // Failed to match fourcc.
285 return distance;
286 }
287
288 // Check resolution and fps.
289 int desired_width = desired.width;
290 int desired_height = desired.height;
Peter Boström0c4e06b2015-10-07 12:23:21 +0200291 int64_t delta_w = supported.width - desired_width;
sergeyu@chromium.org4b26e2e2014-01-15 23:15:54 +0000292 float supported_fps = VideoFormat::IntervalToFpsFloat(supported.interval);
293 float delta_fps =
294 supported_fps - VideoFormat::IntervalToFpsFloat(desired.interval);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000295 // Check height of supported height compared to height we would like it to be.
Peter Boström0c4e06b2015-10-07 12:23:21 +0200296 int64_t aspect_h = desired_width
297 ? supported.width * desired_height / desired_width
298 : desired_height;
299 int64_t delta_h = supported.height - aspect_h;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000300
301 distance = 0;
302 // Set high penalty if the supported format is lower than the desired format.
303 // 3x means we would prefer down to down to 3/4, than up to double.
304 // But we'd prefer up to double than down to 1/2. This is conservative,
305 // strongly avoiding going down in resolution, similar to
306 // the old method, but not completely ruling it out in extreme situations.
307 // It also ignores framerate, which is often very low at high resolutions.
308 // TODO(fbarchard): Improve logic to use weighted factors.
309 static const int kDownPenalty = -3;
310 if (delta_w < 0) {
311 delta_w = delta_w * kDownPenalty;
312 }
313 if (delta_h < 0) {
314 delta_h = delta_h * kDownPenalty;
315 }
316 // Require camera fps to be at least 80% of what is requested if resolution
317 // matches.
318 // Require camera fps to be at least 96% of what is requested, or higher,
319 // if resolution differs. 96% allows for slight variations in fps. e.g. 29.97
320 if (delta_fps < 0) {
sergeyu@chromium.org4b26e2e2014-01-15 23:15:54 +0000321 float min_desirable_fps = delta_w ?
322 VideoFormat::IntervalToFpsFloat(desired.interval) * 28.f / 30.f :
323 VideoFormat::IntervalToFpsFloat(desired.interval) * 23.f / 30.f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000324 delta_fps = -delta_fps;
325 if (supported_fps < min_desirable_fps) {
Peter Boström0c4e06b2015-10-07 12:23:21 +0200326 distance |= static_cast<int64_t>(1) << 62;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000327 } else {
Peter Boström0c4e06b2015-10-07 12:23:21 +0200328 distance |= static_cast<int64_t>(1) << 15;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000329 }
330 }
Peter Boström0c4e06b2015-10-07 12:23:21 +0200331 int64_t idelta_fps = static_cast<int>(delta_fps);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000332
333 // 12 bits for width and height and 8 bits for fps and fourcc.
334 distance |=
sergeyu@chromium.org4b26e2e2014-01-15 23:15:54 +0000335 (delta_w << 28) | (delta_h << 16) | (idelta_fps << 8) | delta_fourcc;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000336
337 return distance;
338}
339
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000340void VideoCapturer::UpdateFilteredSupportedFormats() {
341 filtered_supported_formats_.clear();
342 filtered_supported_formats_ = supported_formats_;
343 if (!max_format_) {
344 return;
345 }
346 std::vector<VideoFormat>::iterator iter = filtered_supported_formats_.begin();
347 while (iter != filtered_supported_formats_.end()) {
348 if (ShouldFilterFormat(*iter)) {
349 iter = filtered_supported_formats_.erase(iter);
350 } else {
351 ++iter;
352 }
353 }
354 if (filtered_supported_formats_.empty()) {
355 // The device only captures at resolutions higher than |max_format_| this
356 // indicates that |max_format_| should be ignored as it is better to capture
357 // at too high a resolution than to not capture at all.
358 filtered_supported_formats_ = supported_formats_;
359 }
360}
361
362bool VideoCapturer::ShouldFilterFormat(const VideoFormat& format) const {
Perfb45d172016-02-29 12:07:35 +0100363 RTC_DCHECK(thread_checker_.CalledOnValidThread());
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000364 if (!enable_camera_list_) {
365 return false;
366 }
367 return format.width > max_format_->width ||
368 format.height > max_format_->height;
369}
370
nisse47ac4622016-05-25 08:47:01 -0700371void VideoCapturer::UpdateInputSize(int width, int height) {
buildbot@webrtc.org0b53bd22014-05-06 17:12:36 +0000372 // Update stats protected from fetches from different thread.
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000373 rtc::CritScope cs(&frame_stats_crit_);
buildbot@webrtc.org0b53bd22014-05-06 17:12:36 +0000374
nissefcc640f2016-04-01 01:10:42 -0700375 input_size_valid_ = true;
nisse47ac4622016-05-25 08:47:01 -0700376 input_width_ = width;
377 input_height_ = height;
henrike@webrtc.org704bf9e2014-02-27 17:52:04 +0000378}
379
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000380} // namespace cricket