blob: 9228e6d84cf75846ce3dc72c96cb96c0c0c58ace [file] [log] [blame]
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00001/*
kjellander1afca732016-02-07 20:46:45 -08002 * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00003 *
kjellander1afca732016-02-07 20:46:45 -08004 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00009 */
henrike@webrtc.org28e20752013-07-10 00:45:36 +000010
kjellandera96e2d72016-02-04 23:52:28 -080011#include "webrtc/media/base/videoadapter.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000012
13#include <limits.h> // For INT_MAX
andresp@webrtc.orgff689be2015-02-12 11:54:26 +000014#include <algorithm>
henrike@webrtc.org28e20752013-07-10 00:45:36 +000015
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000016#include "webrtc/base/logging.h"
17#include "webrtc/base/timeutils.h"
kjellandera96e2d72016-02-04 23:52:28 -080018#include "webrtc/media/base/constants.h"
19#include "webrtc/media/base/videocommon.h"
20#include "webrtc/media/base/videoframe.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000021
22namespace cricket {
23
24// TODO(fbarchard): Make downgrades settable
25static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU.
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +000026// The number of cpu samples to require before adapting. This value depends on
27// the cpu monitor sampling frequency being 2000ms.
28static const int kCpuLoadMinSamples = 3;
henrike@webrtc.org28654cb2013-07-22 21:07:49 +000029// The amount of weight to give to each new cpu load sample. The lower the
30// value, the slower we'll adapt to changing cpu conditions.
31static const float kCpuLoadWeightCoefficient = 0.4f;
32// The seed value for the cpu load moving average.
33static const float kCpuLoadInitialAverage = 0.5f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000034
henrike@webrtc.org28e20752013-07-10 00:45:36 +000035// Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90)
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000036static const float kScaleFactors[] = {
37 1.f / 1.f, // Full size.
38 3.f / 4.f, // 3/4 scale.
39 1.f / 2.f, // 1/2 scale.
40 3.f / 8.f, // 3/8 scale.
41 1.f / 4.f, // 1/4 scale.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000042 3.f / 16.f, // 3/16 scale.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000043 1.f / 8.f, // 1/8 scale.
44 0.f // End of table.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000045};
henrike@webrtc.org28e20752013-07-10 00:45:36 +000046
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000047// TODO(fbarchard): Use this table (optionally) for CPU and GD as well.
48static const float kViewScaleFactors[] = {
49 1.f / 1.f, // Full size.
50 3.f / 4.f, // 3/4 scale.
51 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p.
52 1.f / 2.f, // 1/2 scale.
53 3.f / 8.f, // 3/8 scale.
54 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p.
55 1.f / 4.f, // 1/4 scale.
56 3.f / 16.f, // 3/16 scale.
57 1.f / 8.f, // 1/8 scale.
58 0.f // End of table.
59};
henrike@webrtc.org28e20752013-07-10 00:45:36 +000060
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000061const float* VideoAdapter::GetViewScaleFactors() const {
62 return scale_third_ ? kViewScaleFactors : kScaleFactors;
63}
64
65// For resolutions that would scale down a little instead of up a little,
66// bias toward scaling up a little. This will tend to choose 3/4 scale instead
67// of 2/3 scale, when the 2/3 is not an exact match.
68static const float kUpBias = -0.9f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000069// Find the scale factor that, when applied to width and height, is closest
70// to num_pixels.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000071float VideoAdapter::FindScale(const float* scale_factors,
72 const float upbias,
73 int width, int height,
74 int target_num_pixels) {
75 const float kMinNumPixels = 160 * 90;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000076 if (!target_num_pixels) {
77 return 0.f;
78 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000079 float best_distance = static_cast<float>(INT_MAX);
80 float best_scale = 1.f; // Default to unscaled if nothing matches.
81 float pixels = static_cast<float>(width * height);
82 for (int i = 0; ; ++i) {
83 float scale = scale_factors[i];
84 float test_num_pixels = pixels * scale * scale;
85 // Do not consider scale factors that produce too small images.
86 // Scale factor of 0 at end of table will also exit here.
87 if (test_num_pixels < kMinNumPixels) {
88 break;
89 }
90 float diff = target_num_pixels - test_num_pixels;
91 // If resolution is higher than desired, bias the difference based on
92 // preference for slightly larger for nearest, or avoid completely if
93 // looking for lower resolutions only.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000094 if (diff < 0) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000095 diff = diff * kUpBias;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000096 }
97 if (diff < best_distance) {
98 best_distance = diff;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000099 best_scale = scale;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000100 if (best_distance == 0) { // Found exact match.
101 break;
102 }
103 }
104 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000105 return best_scale;
106}
107
108// Find the closest scale factor.
109float VideoAdapter::FindClosestScale(int width, int height,
110 int target_num_pixels) {
111 return FindScale(kScaleFactors, kUpBias,
112 width, height, target_num_pixels);
113}
114
115// Find the closest view scale factor.
116float VideoAdapter::FindClosestViewScale(int width, int height,
117 int target_num_pixels) {
118 return FindScale(GetViewScaleFactors(), kUpBias,
119 width, height, target_num_pixels);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000120}
121
122// Finds the scale factor that, when applied to width and height, produces
123// fewer than num_pixels.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000124static const float kUpAvoidBias = -1000000000.f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000125float VideoAdapter::FindLowerScale(int width, int height,
126 int target_num_pixels) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000127 return FindScale(GetViewScaleFactors(), kUpAvoidBias,
128 width, height, target_num_pixels);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000129}
130
131// There are several frame sizes used by Adapter. This explains them
132// input_format - set once by server to frame size expected from the camera.
Magnus Jedvertac27e202015-03-24 15:18:39 +0100133// The input frame size is also updated in AdaptFrameResolution.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000134// output_format - size that output would like to be. Includes framerate.
Magnus Jedvertac27e202015-03-24 15:18:39 +0100135// The output frame size is also updated in AdaptFrameResolution.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000136// output_num_pixels - size that output should be constrained to. Used to
137// compute output_format from in_frame.
138// in_frame - actual camera captured frame size, which is typically the same
139// as input_format. This can also be rotated or cropped for aspect ratio.
140// out_frame - actual frame output by adapter. Should be a direct scale of
141// in_frame maintaining rotation and aspect ratio.
142// OnOutputFormatRequest - server requests you send this resolution based on
143// view requests.
144// OnEncoderResolutionRequest - encoder requests you send this resolution based
145// on bandwidth
146// OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
147// cpu load.
148
149///////////////////////////////////////////////////////////////////////
150// Implementation of VideoAdapter
151VideoAdapter::VideoAdapter()
152 : output_num_pixels_(INT_MAX),
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000153 scale_third_(false),
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000154 frames_in_(0),
155 frames_out_(0),
156 frames_scaled_(0),
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000157 adaption_changes_(0),
magjed@webrtc.orga73d7462014-11-14 13:25:25 +0000158 previous_width_(0),
159 previous_height_(0),
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000160 interval_next_frame_(0) {
161}
162
163VideoAdapter::~VideoAdapter() {
164}
165
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000166void VideoAdapter::SetInputFormat(const VideoFormat& format) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000167 rtc::CritScope cs(&critical_section_);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200168 int64_t old_input_interval = input_format_.interval;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000169 input_format_ = format;
andresp@webrtc.orgff689be2015-02-12 11:54:26 +0000170 output_format_.interval =
171 std::max(output_format_.interval, input_format_.interval);
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000172 if (old_input_interval != input_format_.interval) {
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000173 LOG(LS_INFO) << "VAdapt input interval changed from "
174 << old_input_interval << " to " << input_format_.interval;
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000175 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000176}
177
mallinath@webrtc.org1b15f422013-09-06 22:56:28 +0000178void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) {
179 int previous_width = input_format().width;
180 int previous_height = input_format().height;
181 bool is_resolution_change = previous_width > 0 && format.width > 0 &&
182 (previous_width != format.width ||
183 previous_height != format.height);
184 VideoAdapter::SetInputFormat(format);
185 if (is_resolution_change) {
186 int width, height;
187 // Trigger the adaptation logic again, to potentially reset the adaptation
188 // state for things like view requests that may not longer be capping
189 // output (or may now cap output).
190 AdaptToMinimumFormat(&width, &height);
191 LOG(LS_INFO) << "VAdapt Input Resolution Change: "
192 << "Previous input resolution: "
193 << previous_width << "x" << previous_height
194 << " New input resolution: "
195 << format.width << "x" << format.height
196 << " New output resolution: "
197 << width << "x" << height;
198 }
199}
200
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000201void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) {
202 LOG(LS_INFO) << "CPU smoothing is now "
203 << (enable ? "enabled" : "disabled");
204 cpu_smoothing_ = enable;
205}
206
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000207void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000208 rtc::CritScope cs(&critical_section_);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200209 int64_t old_output_interval = output_format_.interval;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000210 output_format_ = format;
211 output_num_pixels_ = output_format_.width * output_format_.height;
andresp@webrtc.orgff689be2015-02-12 11:54:26 +0000212 output_format_.interval =
213 std::max(output_format_.interval, input_format_.interval);
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000214 if (old_output_interval != output_format_.interval) {
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000215 LOG(LS_INFO) << "VAdapt output interval changed from "
216 << old_output_interval << " to " << output_format_.interval;
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000217 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000218}
219
220const VideoFormat& VideoAdapter::input_format() {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000221 rtc::CritScope cs(&critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000222 return input_format_;
223}
224
henrike@webrtc.org704bf9e2014-02-27 17:52:04 +0000225bool VideoAdapter::drops_all_frames() const {
226 return output_num_pixels_ == 0;
227}
228
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000229const VideoFormat& VideoAdapter::output_format() {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000230 rtc::CritScope cs(&critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000231 return output_format_;
232}
233
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000234// Constrain output resolution to this many pixels overall
235void VideoAdapter::SetOutputNumPixels(int num_pixels) {
236 output_num_pixels_ = num_pixels;
237}
238
239int VideoAdapter::GetOutputNumPixels() const {
240 return output_num_pixels_;
241}
242
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000243VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000244 rtc::CritScope cs(&critical_section_);
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000245 ++frames_in_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000246
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000247 SetInputFormat(VideoFormat(
248 in_width, in_height, input_format_.interval, input_format_.fourcc));
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000249
250 // Drop the input frame if necessary.
251 bool should_drop = false;
252 if (!output_num_pixels_) {
253 // Drop all frames as the output format is 0x0.
254 should_drop = true;
255 } else {
256 // Drop some frames based on input fps and output fps.
257 // Normally output fps is less than input fps.
258 // TODO(fbarchard): Consider adjusting interval to reflect the adjusted
259 // interval between frames after dropping some frames.
260 interval_next_frame_ += input_format_.interval;
261 if (output_format_.interval > 0) {
262 if (interval_next_frame_ >= output_format_.interval) {
263 interval_next_frame_ %= output_format_.interval;
264 } else {
265 should_drop = true;
266 }
267 }
268 }
269 if (should_drop) {
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000270 // Show VAdapt log every 90 frames dropped. (3 seconds)
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000271 if ((frames_in_ - frames_out_) % 90 == 0) {
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000272 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
273 // in default calls.
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000274 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
275 << " / out " << frames_out_
276 << " / in " << frames_in_
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000277 << " Changes: " << adaption_changes_
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000278 << " Input: " << in_width
279 << "x" << in_height
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000280 << " i" << input_format_.interval
281 << " Output: i" << output_format_.interval;
282 }
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000283
284 return VideoFormat(); // Drop frame.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000285 }
286
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000287 const float scale = VideoAdapter::FindClosestViewScale(
288 in_width, in_height, output_num_pixels_);
kjellandera96e2d72016-02-04 23:52:28 -0800289 const size_t output_width = static_cast<size_t>(in_width * scale + .5f);
290 const size_t output_height = static_cast<size_t>(in_height * scale + .5f);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000291
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000292 ++frames_out_;
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000293 if (scale != 1)
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000294 ++frames_scaled_;
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000295 // Show VAdapt log every 90 frames output. (3 seconds)
296 // TODO(fbarchard): Consider GetLogSeverity() to change interval to less
297 // for LS_VERBOSE and more for LS_INFO.
298 bool show = (frames_out_) % 90 == 0;
299
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000300 // TODO(fbarchard): LOG the previous output resolution and track input
301 // resolution changes as well. Consider dropping the statistics into their
302 // own class which could be queried publically.
303 bool changed = false;
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000304 if (previous_width_ && (previous_width_ != output_width ||
305 previous_height_ != output_height)) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000306 show = true;
307 ++adaption_changes_;
308 changed = true;
309 }
310 if (show) {
311 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
312 // in default calls.
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000313 LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_
314 << " / out " << frames_out_
315 << " / in " << frames_in_
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000316 << " Changes: " << adaption_changes_
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000317 << " Input: " << in_width
318 << "x" << in_height
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000319 << " i" << input_format_.interval
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000320 << " Scale: " << scale
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000321 << " Output: " << output_width
322 << "x" << output_height
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000323 << " i" << output_format_.interval
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000324 << " Changed: " << (changed ? "true" : "false");
325 }
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000326
327 output_format_.width = output_width;
328 output_format_.height = output_height;
329 previous_width_ = output_width;
330 previous_height_ = output_height;
331
332 return output_format_;
333}
334
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000335void VideoAdapter::set_scale_third(bool enable) {
336 LOG(LS_INFO) << "Video Adapter third scaling is now "
337 << (enable ? "enabled" : "disabled");
338 scale_third_ = enable;
339}
340
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000341///////////////////////////////////////////////////////////////////////
342// Implementation of CoordinatedVideoAdapter
343CoordinatedVideoAdapter::CoordinatedVideoAdapter()
wu@webrtc.org9caf2762013-12-11 18:25:07 +0000344 : cpu_adaptation_(true),
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000345 cpu_smoothing_(false),
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000346 gd_adaptation_(true),
347 view_adaptation_(true),
348 view_switch_(false),
349 cpu_downgrade_count_(0),
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000350 cpu_load_min_samples_(kCpuLoadMinSamples),
351 cpu_load_num_samples_(0),
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000352 high_system_threshold_(kHighSystemCpuThreshold),
353 low_system_threshold_(kLowSystemCpuThreshold),
354 process_threshold_(kProcessCpuThreshold),
355 view_desired_num_pixels_(INT_MAX),
356 view_desired_interval_(0),
357 encoder_desired_num_pixels_(INT_MAX),
358 cpu_desired_num_pixels_(INT_MAX),
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000359 adapt_reason_(ADAPTREASON_NONE),
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000360 system_load_average_(kCpuLoadInitialAverage) {
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000361}
362
363// Helper function to UPGRADE or DOWNGRADE a number of pixels
364void CoordinatedVideoAdapter::StepPixelCount(
365 CoordinatedVideoAdapter::AdaptRequest request,
366 int* num_pixels) {
367 switch (request) {
368 case CoordinatedVideoAdapter::DOWNGRADE:
369 *num_pixels /= 2;
370 break;
371
372 case CoordinatedVideoAdapter::UPGRADE:
373 *num_pixels *= 2;
374 break;
375
376 default: // No change in pixel count
377 break;
378 }
379 return;
380}
381
382// Find the adaptation request of the cpu based on the load. Return UPGRADE if
383// the load is low, DOWNGRADE if the load is high, and KEEP otherwise.
384CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest(
385 int current_cpus, int max_cpus,
386 float process_load, float system_load) {
387 // Downgrade if system is high and plugin is at least more than midrange.
388 if (system_load >= high_system_threshold_ * max_cpus &&
389 process_load >= process_threshold_ * current_cpus) {
390 return CoordinatedVideoAdapter::DOWNGRADE;
391 // Upgrade if system is low.
392 } else if (system_load < low_system_threshold_ * max_cpus) {
393 return CoordinatedVideoAdapter::UPGRADE;
394 }
395 return CoordinatedVideoAdapter::KEEP;
396}
397
398// A remote view request for a new resolution.
399void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000400 rtc::CritScope cs(&request_critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000401 if (!view_adaptation_) {
402 return;
403 }
404 // Set output for initial aspect ratio in mediachannel unittests.
405 int old_num_pixels = GetOutputNumPixels();
406 SetOutputFormat(format);
407 SetOutputNumPixels(old_num_pixels);
408 view_desired_num_pixels_ = format.width * format.height;
409 view_desired_interval_ = format.interval;
410 int new_width, new_height;
411 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
412 LOG(LS_INFO) << "VAdapt View Request: "
413 << format.width << "x" << format.height
414 << " Pixels: " << view_desired_num_pixels_
415 << " Changed: " << (changed ? "true" : "false")
416 << " To: " << new_width << "x" << new_height;
417}
418
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000419void CoordinatedVideoAdapter::set_cpu_load_min_samples(
420 int cpu_load_min_samples) {
421 if (cpu_load_min_samples_ != cpu_load_min_samples) {
422 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: "
423 << cpu_load_min_samples_ << " to "
424 << cpu_load_min_samples;
425 cpu_load_min_samples_ = cpu_load_min_samples;
426 }
427}
428
429void CoordinatedVideoAdapter::set_high_system_threshold(
430 float high_system_threshold) {
431 ASSERT(high_system_threshold <= 1.0f);
432 ASSERT(high_system_threshold >= 0.0f);
433 if (high_system_threshold_ != high_system_threshold) {
434 LOG(LS_INFO) << "VAdapt Change High System Threshold from: "
435 << high_system_threshold_ << " to " << high_system_threshold;
436 high_system_threshold_ = high_system_threshold;
437 }
438}
439
440void CoordinatedVideoAdapter::set_low_system_threshold(
441 float low_system_threshold) {
442 ASSERT(low_system_threshold <= 1.0f);
443 ASSERT(low_system_threshold >= 0.0f);
444 if (low_system_threshold_ != low_system_threshold) {
445 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: "
446 << low_system_threshold_ << " to " << low_system_threshold;
447 low_system_threshold_ = low_system_threshold;
448 }
449}
450
451void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) {
452 ASSERT(process_threshold <= 1.0f);
453 ASSERT(process_threshold >= 0.0f);
454 if (process_threshold_ != process_threshold) {
455 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: "
456 << process_threshold_ << " to " << process_threshold;
457 process_threshold_ = process_threshold;
458 }
459}
460
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000461// A Bandwidth GD request for new resolution
462void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
463 int width, int height, AdaptRequest request) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000464 rtc::CritScope cs(&request_critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000465 if (!gd_adaptation_) {
466 return;
467 }
468 int old_encoder_desired_num_pixels = encoder_desired_num_pixels_;
469 if (KEEP != request) {
470 int new_encoder_desired_num_pixels = width * height;
471 int old_num_pixels = GetOutputNumPixels();
472 if (new_encoder_desired_num_pixels != old_num_pixels) {
473 LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored";
474 } else {
475 // Update the encoder desired format based on the request.
476 encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
477 StepPixelCount(request, &encoder_desired_num_pixels_);
478 }
479 }
480 int new_width, new_height;
481 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
482
483 // Ignore up or keep if no change.
484 if (DOWNGRADE != request && view_switch_ && !changed) {
485 encoder_desired_num_pixels_ = old_encoder_desired_num_pixels;
486 LOG(LS_VERBOSE) << "VAdapt ignoring GD request.";
487 }
488
489 LOG(LS_INFO) << "VAdapt GD Request: "
490 << (DOWNGRADE == request ? "down" :
491 (UPGRADE == request ? "up" : "keep"))
492 << " From: " << width << "x" << height
493 << " Pixels: " << encoder_desired_num_pixels_
494 << " Changed: " << (changed ? "true" : "false")
495 << " To: " << new_width << "x" << new_height;
496}
497
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000498// A Bandwidth GD request for new resolution
499void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000500 rtc::CritScope cs(&request_critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000501 if (!cpu_adaptation_) {
502 return;
503 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000504 // Update how many times we have downgraded due to the cpu load.
505 switch (request) {
506 case DOWNGRADE:
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000507 // Ignore downgrades if we have downgraded the maximum times.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000508 if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000509 ++cpu_downgrade_count_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000510 } else {
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000511 LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
512 "because maximum downgrades reached";
513 SignalCpuAdaptationUnable();
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000514 }
515 break;
516 case UPGRADE:
517 if (cpu_downgrade_count_ > 0) {
518 bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
519 if (is_min) {
520 --cpu_downgrade_count_;
521 } else {
522 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
523 "because cpu is not limiting resolution";
524 }
525 } else {
526 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
527 "because minimum downgrades reached";
528 }
529 break;
530 case KEEP:
531 default:
532 break;
533 }
534 if (KEEP != request) {
535 // TODO(fbarchard): compute stepping up/down from OutputNumPixels but
536 // clamp to inputpixels / 4 (2 steps)
537 cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX :
538 static_cast<int>(input_format().width * input_format().height >>
539 cpu_downgrade_count_);
540 }
541 int new_width, new_height;
542 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
543 LOG(LS_INFO) << "VAdapt CPU Request: "
544 << (DOWNGRADE == request ? "down" :
545 (UPGRADE == request ? "up" : "keep"))
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000546 << " Steps: " << cpu_downgrade_count_
547 << " Changed: " << (changed ? "true" : "false")
548 << " To: " << new_width << "x" << new_height;
549}
550
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000551// A CPU request for new resolution
552// TODO(fbarchard): Move outside adapter.
553void CoordinatedVideoAdapter::OnCpuLoadUpdated(
554 int current_cpus, int max_cpus, float process_load, float system_load) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000555 rtc::CritScope cs(&request_critical_section_);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000556 if (!cpu_adaptation_) {
557 return;
558 }
559 // Update the moving average of system load. Even if we aren't smoothing,
560 // we'll still calculate this information, in case smoothing is later enabled.
561 system_load_average_ = kCpuLoadWeightCoefficient * system_load +
562 (1.0f - kCpuLoadWeightCoefficient) * system_load_average_;
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000563 ++cpu_load_num_samples_;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000564 if (cpu_smoothing_) {
565 system_load = system_load_average_;
566 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000567 AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
568 process_load, system_load);
569 // Make sure we're not adapting too quickly.
570 if (request != KEEP) {
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000571 if (cpu_load_num_samples_ < cpu_load_min_samples_) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000572 LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until "
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000573 << (cpu_load_min_samples_ - cpu_load_num_samples_)
574 << " more samples";
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000575 request = KEEP;
576 }
577 }
578
579 OnCpuResolutionRequest(request);
580}
581
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000582// Called by cpu adapter on up requests.
583bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
584 // Find closest scale factor that matches input resolution to min_num_pixels
585 // and set that for output resolution. This is not needed for VideoAdapter,
586 // but provides feedback to unittests and users on expected resolution.
587 // Actual resolution is based on input frame.
588 VideoFormat new_output = output_format();
589 VideoFormat input = input_format();
590 if (input_format().IsSize0x0()) {
591 input = new_output;
592 }
593 float scale = 1.0f;
594 if (!input.IsSize0x0()) {
595 scale = FindClosestScale(input.width,
596 input.height,
597 pixels);
598 }
599 new_output.width = static_cast<int>(input.width * scale + .5f);
600 new_output.height = static_cast<int>(input.height * scale + .5f);
601 int new_pixels = new_output.width * new_output.height;
602 int num_pixels = GetOutputNumPixels();
603 return new_pixels <= num_pixels;
604}
605
606// Called by all coordinators when there is a change.
607bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width,
608 int* new_height) {
609 VideoFormat new_output = output_format();
610 VideoFormat input = input_format();
611 if (input_format().IsSize0x0()) {
612 input = new_output;
613 }
614 int old_num_pixels = GetOutputNumPixels();
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000615 int min_num_pixels = INT_MAX;
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000616 adapt_reason_ = ADAPTREASON_NONE;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000617
618 // Reduce resolution based on encoder bandwidth (GD).
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000619 if (encoder_desired_num_pixels_ &&
620 (encoder_desired_num_pixels_ < min_num_pixels)) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000621 adapt_reason_ |= ADAPTREASON_BANDWIDTH;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000622 min_num_pixels = encoder_desired_num_pixels_;
623 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000624 // Reduce resolution based on CPU.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000625 if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000626 (cpu_desired_num_pixels_ <= min_num_pixels)) {
627 if (cpu_desired_num_pixels_ < min_num_pixels) {
628 adapt_reason_ = ADAPTREASON_CPU;
629 } else {
630 adapt_reason_ |= ADAPTREASON_CPU;
631 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000632 min_num_pixels = cpu_desired_num_pixels_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000633 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000634 // Round resolution for GD or CPU to allow 1/2 to map to 9/16.
635 if (!input.IsSize0x0() && min_num_pixels != INT_MAX) {
636 float scale = FindClosestScale(input.width, input.height, min_num_pixels);
637 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
638 static_cast<int>(input.height * scale + .5f);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000639 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000640 // Reduce resolution based on View Request.
641 if (view_desired_num_pixels_ <= min_num_pixels) {
642 if (view_desired_num_pixels_ < min_num_pixels) {
643 adapt_reason_ = ADAPTREASON_VIEW;
644 } else {
645 adapt_reason_ |= ADAPTREASON_VIEW;
646 }
647 min_num_pixels = view_desired_num_pixels_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000648 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000649 // Snap to a scale factor.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000650 float scale = 1.0f;
651 if (!input.IsSize0x0()) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000652 scale = FindLowerScale(input.width, input.height, min_num_pixels);
653 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
654 static_cast<int>(input.height * scale + .5f);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000655 }
656 if (scale == 1.0f) {
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000657 adapt_reason_ = ADAPTREASON_NONE;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000658 }
659 *new_width = new_output.width = static_cast<int>(input.width * scale + .5f);
660 *new_height = new_output.height = static_cast<int>(input.height * scale +
661 .5f);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000662 SetOutputNumPixels(min_num_pixels);
663
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000664 new_output.interval = view_desired_interval_;
665 SetOutputFormat(new_output);
666 int new_num_pixels = GetOutputNumPixels();
667 bool changed = new_num_pixels != old_num_pixels;
668
669 static const char* kReasons[8] = {
670 "None",
671 "CPU",
672 "BANDWIDTH",
673 "CPU+BANDWIDTH",
674 "VIEW",
675 "CPU+VIEW",
676 "BANDWIDTH+VIEW",
677 "CPU+BANDWIDTH+VIEW",
678 };
679
680 LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
681 << " GD: " << encoder_desired_num_pixels_
682 << " CPU: " << cpu_desired_num_pixels_
683 << " Pixels: " << min_num_pixels
684 << " Input: " << input.width
685 << "x" << input.height
686 << " Scale: " << scale
687 << " Resolution: " << new_output.width
688 << "x" << new_output.height
689 << " Changed: " << (changed ? "true" : "false")
690 << " Reason: " << kReasons[adapt_reason_];
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000691
692 if (changed) {
693 // When any adaptation occurs, historic CPU load levels are no longer
694 // accurate. Clear out our state so we can re-learn at the new normal.
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000695 cpu_load_num_samples_ = 0;
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000696 system_load_average_ = kCpuLoadInitialAverage;
697 }
698
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000699 return changed;
700}
701
702} // namespace cricket