blob: 89a4493108edc1d56661942fe8c2b0e4ec7733f6 [file] [log] [blame]
jlmiller@webrtc.org5f93d0a2015-01-20 21:36:13 +00001/*
2 * libjingle
3 * Copyright 2010 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
henrike@webrtc.org28e20752013-07-10 00:45:36 +000027
kjellandera96e2d72016-02-04 23:52:28 -080028#include "webrtc/media/base/videoadapter.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000029
30#include <limits.h> // For INT_MAX
andresp@webrtc.orgff689be2015-02-12 11:54:26 +000031#include <algorithm>
henrike@webrtc.org28e20752013-07-10 00:45:36 +000032
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000033#include "webrtc/base/logging.h"
34#include "webrtc/base/timeutils.h"
kjellandera96e2d72016-02-04 23:52:28 -080035#include "webrtc/media/base/constants.h"
36#include "webrtc/media/base/videocommon.h"
37#include "webrtc/media/base/videoframe.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000038
39namespace cricket {
40
41// TODO(fbarchard): Make downgrades settable
42static const int kMaxCpuDowngrades = 2; // Downgrade at most 2 times for CPU.
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +000043// The number of cpu samples to require before adapting. This value depends on
44// the cpu monitor sampling frequency being 2000ms.
45static const int kCpuLoadMinSamples = 3;
henrike@webrtc.org28654cb2013-07-22 21:07:49 +000046// The amount of weight to give to each new cpu load sample. The lower the
47// value, the slower we'll adapt to changing cpu conditions.
48static const float kCpuLoadWeightCoefficient = 0.4f;
49// The seed value for the cpu load moving average.
50static const float kCpuLoadInitialAverage = 0.5f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000051
henrike@webrtc.org28e20752013-07-10 00:45:36 +000052// Desktop needs 1/8 scale for HD (1280 x 720) to QQVGA (160 x 90)
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000053static const float kScaleFactors[] = {
54 1.f / 1.f, // Full size.
55 3.f / 4.f, // 3/4 scale.
56 1.f / 2.f, // 1/2 scale.
57 3.f / 8.f, // 3/8 scale.
58 1.f / 4.f, // 1/4 scale.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000059 3.f / 16.f, // 3/16 scale.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000060 1.f / 8.f, // 1/8 scale.
61 0.f // End of table.
henrike@webrtc.org28e20752013-07-10 00:45:36 +000062};
henrike@webrtc.org28e20752013-07-10 00:45:36 +000063
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000064// TODO(fbarchard): Use this table (optionally) for CPU and GD as well.
65static const float kViewScaleFactors[] = {
66 1.f / 1.f, // Full size.
67 3.f / 4.f, // 3/4 scale.
68 2.f / 3.f, // 2/3 scale. // Allow 1080p to 720p.
69 1.f / 2.f, // 1/2 scale.
70 3.f / 8.f, // 3/8 scale.
71 1.f / 3.f, // 1/3 scale. // Allow 1080p to 360p.
72 1.f / 4.f, // 1/4 scale.
73 3.f / 16.f, // 3/16 scale.
74 1.f / 8.f, // 1/8 scale.
75 0.f // End of table.
76};
henrike@webrtc.org28e20752013-07-10 00:45:36 +000077
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000078const float* VideoAdapter::GetViewScaleFactors() const {
79 return scale_third_ ? kViewScaleFactors : kScaleFactors;
80}
81
82// For resolutions that would scale down a little instead of up a little,
83// bias toward scaling up a little. This will tend to choose 3/4 scale instead
84// of 2/3 scale, when the 2/3 is not an exact match.
85static const float kUpBias = -0.9f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000086// Find the scale factor that, when applied to width and height, is closest
87// to num_pixels.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000088float VideoAdapter::FindScale(const float* scale_factors,
89 const float upbias,
90 int width, int height,
91 int target_num_pixels) {
92 const float kMinNumPixels = 160 * 90;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000093 if (!target_num_pixels) {
94 return 0.f;
95 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000096 float best_distance = static_cast<float>(INT_MAX);
97 float best_scale = 1.f; // Default to unscaled if nothing matches.
98 float pixels = static_cast<float>(width * height);
99 for (int i = 0; ; ++i) {
100 float scale = scale_factors[i];
101 float test_num_pixels = pixels * scale * scale;
102 // Do not consider scale factors that produce too small images.
103 // Scale factor of 0 at end of table will also exit here.
104 if (test_num_pixels < kMinNumPixels) {
105 break;
106 }
107 float diff = target_num_pixels - test_num_pixels;
108 // If resolution is higher than desired, bias the difference based on
109 // preference for slightly larger for nearest, or avoid completely if
110 // looking for lower resolutions only.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000111 if (diff < 0) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000112 diff = diff * kUpBias;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000113 }
114 if (diff < best_distance) {
115 best_distance = diff;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000116 best_scale = scale;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000117 if (best_distance == 0) { // Found exact match.
118 break;
119 }
120 }
121 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000122 return best_scale;
123}
124
125// Find the closest scale factor.
126float VideoAdapter::FindClosestScale(int width, int height,
127 int target_num_pixels) {
128 return FindScale(kScaleFactors, kUpBias,
129 width, height, target_num_pixels);
130}
131
132// Find the closest view scale factor.
133float VideoAdapter::FindClosestViewScale(int width, int height,
134 int target_num_pixels) {
135 return FindScale(GetViewScaleFactors(), kUpBias,
136 width, height, target_num_pixels);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000137}
138
139// Finds the scale factor that, when applied to width and height, produces
140// fewer than num_pixels.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000141static const float kUpAvoidBias = -1000000000.f;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000142float VideoAdapter::FindLowerScale(int width, int height,
143 int target_num_pixels) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000144 return FindScale(GetViewScaleFactors(), kUpAvoidBias,
145 width, height, target_num_pixels);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000146}
147
148// There are several frame sizes used by Adapter. This explains them
149// input_format - set once by server to frame size expected from the camera.
Magnus Jedvertac27e202015-03-24 15:18:39 +0100150// The input frame size is also updated in AdaptFrameResolution.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000151// output_format - size that output would like to be. Includes framerate.
Magnus Jedvertac27e202015-03-24 15:18:39 +0100152// The output frame size is also updated in AdaptFrameResolution.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000153// output_num_pixels - size that output should be constrained to. Used to
154// compute output_format from in_frame.
155// in_frame - actual camera captured frame size, which is typically the same
156// as input_format. This can also be rotated or cropped for aspect ratio.
157// out_frame - actual frame output by adapter. Should be a direct scale of
158// in_frame maintaining rotation and aspect ratio.
159// OnOutputFormatRequest - server requests you send this resolution based on
160// view requests.
161// OnEncoderResolutionRequest - encoder requests you send this resolution based
162// on bandwidth
163// OnCpuLoadUpdated - cpu monitor requests you send this resolution based on
164// cpu load.
165
166///////////////////////////////////////////////////////////////////////
167// Implementation of VideoAdapter
168VideoAdapter::VideoAdapter()
169 : output_num_pixels_(INT_MAX),
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000170 scale_third_(false),
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000171 frames_in_(0),
172 frames_out_(0),
173 frames_scaled_(0),
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000174 adaption_changes_(0),
magjed@webrtc.orga73d7462014-11-14 13:25:25 +0000175 previous_width_(0),
176 previous_height_(0),
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000177 interval_next_frame_(0) {
178}
179
180VideoAdapter::~VideoAdapter() {
181}
182
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000183void VideoAdapter::SetInputFormat(const VideoFormat& format) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000184 rtc::CritScope cs(&critical_section_);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200185 int64_t old_input_interval = input_format_.interval;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000186 input_format_ = format;
andresp@webrtc.orgff689be2015-02-12 11:54:26 +0000187 output_format_.interval =
188 std::max(output_format_.interval, input_format_.interval);
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000189 if (old_input_interval != input_format_.interval) {
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000190 LOG(LS_INFO) << "VAdapt input interval changed from "
191 << old_input_interval << " to " << input_format_.interval;
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000192 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000193}
194
mallinath@webrtc.org1b15f422013-09-06 22:56:28 +0000195void CoordinatedVideoAdapter::SetInputFormat(const VideoFormat& format) {
196 int previous_width = input_format().width;
197 int previous_height = input_format().height;
198 bool is_resolution_change = previous_width > 0 && format.width > 0 &&
199 (previous_width != format.width ||
200 previous_height != format.height);
201 VideoAdapter::SetInputFormat(format);
202 if (is_resolution_change) {
203 int width, height;
204 // Trigger the adaptation logic again, to potentially reset the adaptation
205 // state for things like view requests that may not longer be capping
206 // output (or may now cap output).
207 AdaptToMinimumFormat(&width, &height);
208 LOG(LS_INFO) << "VAdapt Input Resolution Change: "
209 << "Previous input resolution: "
210 << previous_width << "x" << previous_height
211 << " New input resolution: "
212 << format.width << "x" << format.height
213 << " New output resolution: "
214 << width << "x" << height;
215 }
216}
217
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000218void CoordinatedVideoAdapter::set_cpu_smoothing(bool enable) {
219 LOG(LS_INFO) << "CPU smoothing is now "
220 << (enable ? "enabled" : "disabled");
221 cpu_smoothing_ = enable;
222}
223
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000224void VideoAdapter::SetOutputFormat(const VideoFormat& format) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000225 rtc::CritScope cs(&critical_section_);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200226 int64_t old_output_interval = output_format_.interval;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000227 output_format_ = format;
228 output_num_pixels_ = output_format_.width * output_format_.height;
andresp@webrtc.orgff689be2015-02-12 11:54:26 +0000229 output_format_.interval =
230 std::max(output_format_.interval, input_format_.interval);
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000231 if (old_output_interval != output_format_.interval) {
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000232 LOG(LS_INFO) << "VAdapt output interval changed from "
233 << old_output_interval << " to " << output_format_.interval;
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000234 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000235}
236
237const VideoFormat& VideoAdapter::input_format() {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000238 rtc::CritScope cs(&critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000239 return input_format_;
240}
241
henrike@webrtc.org704bf9e2014-02-27 17:52:04 +0000242bool VideoAdapter::drops_all_frames() const {
243 return output_num_pixels_ == 0;
244}
245
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000246const VideoFormat& VideoAdapter::output_format() {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000247 rtc::CritScope cs(&critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000248 return output_format_;
249}
250
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000251// Constrain output resolution to this many pixels overall
252void VideoAdapter::SetOutputNumPixels(int num_pixels) {
253 output_num_pixels_ = num_pixels;
254}
255
256int VideoAdapter::GetOutputNumPixels() const {
257 return output_num_pixels_;
258}
259
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000260VideoFormat VideoAdapter::AdaptFrameResolution(int in_width, int in_height) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000261 rtc::CritScope cs(&critical_section_);
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000262 ++frames_in_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000263
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000264 SetInputFormat(VideoFormat(
265 in_width, in_height, input_format_.interval, input_format_.fourcc));
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000266
267 // Drop the input frame if necessary.
268 bool should_drop = false;
269 if (!output_num_pixels_) {
270 // Drop all frames as the output format is 0x0.
271 should_drop = true;
272 } else {
273 // Drop some frames based on input fps and output fps.
274 // Normally output fps is less than input fps.
275 // TODO(fbarchard): Consider adjusting interval to reflect the adjusted
276 // interval between frames after dropping some frames.
277 interval_next_frame_ += input_format_.interval;
278 if (output_format_.interval > 0) {
279 if (interval_next_frame_ >= output_format_.interval) {
280 interval_next_frame_ %= output_format_.interval;
281 } else {
282 should_drop = true;
283 }
284 }
285 }
286 if (should_drop) {
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000287 // Show VAdapt log every 90 frames dropped. (3 seconds)
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000288 if ((frames_in_ - frames_out_) % 90 == 0) {
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000289 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
290 // in default calls.
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000291 LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
292 << " / out " << frames_out_
293 << " / in " << frames_in_
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000294 << " Changes: " << adaption_changes_
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000295 << " Input: " << in_width
296 << "x" << in_height
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000297 << " i" << input_format_.interval
298 << " Output: i" << output_format_.interval;
299 }
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000300
301 return VideoFormat(); // Drop frame.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000302 }
303
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000304 const float scale = VideoAdapter::FindClosestViewScale(
305 in_width, in_height, output_num_pixels_);
kjellandera96e2d72016-02-04 23:52:28 -0800306 const size_t output_width = static_cast<size_t>(in_width * scale + .5f);
307 const size_t output_height = static_cast<size_t>(in_height * scale + .5f);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000308
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000309 ++frames_out_;
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000310 if (scale != 1)
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000311 ++frames_scaled_;
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000312 // Show VAdapt log every 90 frames output. (3 seconds)
313 // TODO(fbarchard): Consider GetLogSeverity() to change interval to less
314 // for LS_VERBOSE and more for LS_INFO.
315 bool show = (frames_out_) % 90 == 0;
316
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000317 // TODO(fbarchard): LOG the previous output resolution and track input
318 // resolution changes as well. Consider dropping the statistics into their
319 // own class which could be queried publically.
320 bool changed = false;
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000321 if (previous_width_ && (previous_width_ != output_width ||
322 previous_height_ != output_height)) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000323 show = true;
324 ++adaption_changes_;
325 changed = true;
326 }
327 if (show) {
328 // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
329 // in default calls.
wu@webrtc.orgb9a088b2014-02-13 23:18:49 +0000330 LOG(LS_INFO) << "VAdapt Frame: scaled " << frames_scaled_
331 << " / out " << frames_out_
332 << " / in " << frames_in_
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000333 << " Changes: " << adaption_changes_
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000334 << " Input: " << in_width
335 << "x" << in_height
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000336 << " i" << input_format_.interval
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000337 << " Scale: " << scale
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000338 << " Output: " << output_width
339 << "x" << output_height
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +0000340 << " i" << output_format_.interval
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000341 << " Changed: " << (changed ? "true" : "false");
342 }
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +0000343
344 output_format_.width = output_width;
345 output_format_.height = output_height;
346 previous_width_ = output_width;
347 previous_height_ = output_height;
348
349 return output_format_;
350}
351
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000352void VideoAdapter::set_scale_third(bool enable) {
353 LOG(LS_INFO) << "Video Adapter third scaling is now "
354 << (enable ? "enabled" : "disabled");
355 scale_third_ = enable;
356}
357
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000358///////////////////////////////////////////////////////////////////////
359// Implementation of CoordinatedVideoAdapter
360CoordinatedVideoAdapter::CoordinatedVideoAdapter()
wu@webrtc.org9caf2762013-12-11 18:25:07 +0000361 : cpu_adaptation_(true),
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000362 cpu_smoothing_(false),
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000363 gd_adaptation_(true),
364 view_adaptation_(true),
365 view_switch_(false),
366 cpu_downgrade_count_(0),
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000367 cpu_load_min_samples_(kCpuLoadMinSamples),
368 cpu_load_num_samples_(0),
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000369 high_system_threshold_(kHighSystemCpuThreshold),
370 low_system_threshold_(kLowSystemCpuThreshold),
371 process_threshold_(kProcessCpuThreshold),
372 view_desired_num_pixels_(INT_MAX),
373 view_desired_interval_(0),
374 encoder_desired_num_pixels_(INT_MAX),
375 cpu_desired_num_pixels_(INT_MAX),
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000376 adapt_reason_(ADAPTREASON_NONE),
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000377 system_load_average_(kCpuLoadInitialAverage) {
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000378}
379
380// Helper function to UPGRADE or DOWNGRADE a number of pixels
381void CoordinatedVideoAdapter::StepPixelCount(
382 CoordinatedVideoAdapter::AdaptRequest request,
383 int* num_pixels) {
384 switch (request) {
385 case CoordinatedVideoAdapter::DOWNGRADE:
386 *num_pixels /= 2;
387 break;
388
389 case CoordinatedVideoAdapter::UPGRADE:
390 *num_pixels *= 2;
391 break;
392
393 default: // No change in pixel count
394 break;
395 }
396 return;
397}
398
399// Find the adaptation request of the cpu based on the load. Return UPGRADE if
400// the load is low, DOWNGRADE if the load is high, and KEEP otherwise.
401CoordinatedVideoAdapter::AdaptRequest CoordinatedVideoAdapter::FindCpuRequest(
402 int current_cpus, int max_cpus,
403 float process_load, float system_load) {
404 // Downgrade if system is high and plugin is at least more than midrange.
405 if (system_load >= high_system_threshold_ * max_cpus &&
406 process_load >= process_threshold_ * current_cpus) {
407 return CoordinatedVideoAdapter::DOWNGRADE;
408 // Upgrade if system is low.
409 } else if (system_load < low_system_threshold_ * max_cpus) {
410 return CoordinatedVideoAdapter::UPGRADE;
411 }
412 return CoordinatedVideoAdapter::KEEP;
413}
414
415// A remote view request for a new resolution.
416void CoordinatedVideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000417 rtc::CritScope cs(&request_critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000418 if (!view_adaptation_) {
419 return;
420 }
421 // Set output for initial aspect ratio in mediachannel unittests.
422 int old_num_pixels = GetOutputNumPixels();
423 SetOutputFormat(format);
424 SetOutputNumPixels(old_num_pixels);
425 view_desired_num_pixels_ = format.width * format.height;
426 view_desired_interval_ = format.interval;
427 int new_width, new_height;
428 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
429 LOG(LS_INFO) << "VAdapt View Request: "
430 << format.width << "x" << format.height
431 << " Pixels: " << view_desired_num_pixels_
432 << " Changed: " << (changed ? "true" : "false")
433 << " To: " << new_width << "x" << new_height;
434}
435
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000436void CoordinatedVideoAdapter::set_cpu_load_min_samples(
437 int cpu_load_min_samples) {
438 if (cpu_load_min_samples_ != cpu_load_min_samples) {
439 LOG(LS_INFO) << "VAdapt Change Cpu Adapt Min Samples from: "
440 << cpu_load_min_samples_ << " to "
441 << cpu_load_min_samples;
442 cpu_load_min_samples_ = cpu_load_min_samples;
443 }
444}
445
446void CoordinatedVideoAdapter::set_high_system_threshold(
447 float high_system_threshold) {
448 ASSERT(high_system_threshold <= 1.0f);
449 ASSERT(high_system_threshold >= 0.0f);
450 if (high_system_threshold_ != high_system_threshold) {
451 LOG(LS_INFO) << "VAdapt Change High System Threshold from: "
452 << high_system_threshold_ << " to " << high_system_threshold;
453 high_system_threshold_ = high_system_threshold;
454 }
455}
456
457void CoordinatedVideoAdapter::set_low_system_threshold(
458 float low_system_threshold) {
459 ASSERT(low_system_threshold <= 1.0f);
460 ASSERT(low_system_threshold >= 0.0f);
461 if (low_system_threshold_ != low_system_threshold) {
462 LOG(LS_INFO) << "VAdapt Change Low System Threshold from: "
463 << low_system_threshold_ << " to " << low_system_threshold;
464 low_system_threshold_ = low_system_threshold;
465 }
466}
467
468void CoordinatedVideoAdapter::set_process_threshold(float process_threshold) {
469 ASSERT(process_threshold <= 1.0f);
470 ASSERT(process_threshold >= 0.0f);
471 if (process_threshold_ != process_threshold) {
472 LOG(LS_INFO) << "VAdapt Change High Process Threshold from: "
473 << process_threshold_ << " to " << process_threshold;
474 process_threshold_ = process_threshold;
475 }
476}
477
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000478// A Bandwidth GD request for new resolution
479void CoordinatedVideoAdapter::OnEncoderResolutionRequest(
480 int width, int height, AdaptRequest request) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000481 rtc::CritScope cs(&request_critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000482 if (!gd_adaptation_) {
483 return;
484 }
485 int old_encoder_desired_num_pixels = encoder_desired_num_pixels_;
486 if (KEEP != request) {
487 int new_encoder_desired_num_pixels = width * height;
488 int old_num_pixels = GetOutputNumPixels();
489 if (new_encoder_desired_num_pixels != old_num_pixels) {
490 LOG(LS_VERBOSE) << "VAdapt GD resolution stale. Ignored";
491 } else {
492 // Update the encoder desired format based on the request.
493 encoder_desired_num_pixels_ = new_encoder_desired_num_pixels;
494 StepPixelCount(request, &encoder_desired_num_pixels_);
495 }
496 }
497 int new_width, new_height;
498 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
499
500 // Ignore up or keep if no change.
501 if (DOWNGRADE != request && view_switch_ && !changed) {
502 encoder_desired_num_pixels_ = old_encoder_desired_num_pixels;
503 LOG(LS_VERBOSE) << "VAdapt ignoring GD request.";
504 }
505
506 LOG(LS_INFO) << "VAdapt GD Request: "
507 << (DOWNGRADE == request ? "down" :
508 (UPGRADE == request ? "up" : "keep"))
509 << " From: " << width << "x" << height
510 << " Pixels: " << encoder_desired_num_pixels_
511 << " Changed: " << (changed ? "true" : "false")
512 << " To: " << new_width << "x" << new_height;
513}
514
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000515// A Bandwidth GD request for new resolution
516void CoordinatedVideoAdapter::OnCpuResolutionRequest(AdaptRequest request) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000517 rtc::CritScope cs(&request_critical_section_);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000518 if (!cpu_adaptation_) {
519 return;
520 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000521 // Update how many times we have downgraded due to the cpu load.
522 switch (request) {
523 case DOWNGRADE:
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000524 // Ignore downgrades if we have downgraded the maximum times.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000525 if (cpu_downgrade_count_ < kMaxCpuDowngrades) {
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000526 ++cpu_downgrade_count_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000527 } else {
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000528 LOG(LS_VERBOSE) << "VAdapt CPU load high but do not downgrade "
529 "because maximum downgrades reached";
530 SignalCpuAdaptationUnable();
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000531 }
532 break;
533 case UPGRADE:
534 if (cpu_downgrade_count_ > 0) {
535 bool is_min = IsMinimumFormat(cpu_desired_num_pixels_);
536 if (is_min) {
537 --cpu_downgrade_count_;
538 } else {
539 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
540 "because cpu is not limiting resolution";
541 }
542 } else {
543 LOG(LS_VERBOSE) << "VAdapt CPU load low but do not upgrade "
544 "because minimum downgrades reached";
545 }
546 break;
547 case KEEP:
548 default:
549 break;
550 }
551 if (KEEP != request) {
552 // TODO(fbarchard): compute stepping up/down from OutputNumPixels but
553 // clamp to inputpixels / 4 (2 steps)
554 cpu_desired_num_pixels_ = cpu_downgrade_count_ == 0 ? INT_MAX :
555 static_cast<int>(input_format().width * input_format().height >>
556 cpu_downgrade_count_);
557 }
558 int new_width, new_height;
559 bool changed = AdaptToMinimumFormat(&new_width, &new_height);
560 LOG(LS_INFO) << "VAdapt CPU Request: "
561 << (DOWNGRADE == request ? "down" :
562 (UPGRADE == request ? "up" : "keep"))
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000563 << " Steps: " << cpu_downgrade_count_
564 << " Changed: " << (changed ? "true" : "false")
565 << " To: " << new_width << "x" << new_height;
566}
567
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000568// A CPU request for new resolution
569// TODO(fbarchard): Move outside adapter.
570void CoordinatedVideoAdapter::OnCpuLoadUpdated(
571 int current_cpus, int max_cpus, float process_load, float system_load) {
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000572 rtc::CritScope cs(&request_critical_section_);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000573 if (!cpu_adaptation_) {
574 return;
575 }
576 // Update the moving average of system load. Even if we aren't smoothing,
577 // we'll still calculate this information, in case smoothing is later enabled.
578 system_load_average_ = kCpuLoadWeightCoefficient * system_load +
579 (1.0f - kCpuLoadWeightCoefficient) * system_load_average_;
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000580 ++cpu_load_num_samples_;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000581 if (cpu_smoothing_) {
582 system_load = system_load_average_;
583 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000584 AdaptRequest request = FindCpuRequest(current_cpus, max_cpus,
585 process_load, system_load);
586 // Make sure we're not adapting too quickly.
587 if (request != KEEP) {
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000588 if (cpu_load_num_samples_ < cpu_load_min_samples_) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000589 LOG(LS_VERBOSE) << "VAdapt CPU load high/low but do not adapt until "
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000590 << (cpu_load_min_samples_ - cpu_load_num_samples_)
591 << " more samples";
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000592 request = KEEP;
593 }
594 }
595
596 OnCpuResolutionRequest(request);
597}
598
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000599// Called by cpu adapter on up requests.
600bool CoordinatedVideoAdapter::IsMinimumFormat(int pixels) {
601 // Find closest scale factor that matches input resolution to min_num_pixels
602 // and set that for output resolution. This is not needed for VideoAdapter,
603 // but provides feedback to unittests and users on expected resolution.
604 // Actual resolution is based on input frame.
605 VideoFormat new_output = output_format();
606 VideoFormat input = input_format();
607 if (input_format().IsSize0x0()) {
608 input = new_output;
609 }
610 float scale = 1.0f;
611 if (!input.IsSize0x0()) {
612 scale = FindClosestScale(input.width,
613 input.height,
614 pixels);
615 }
616 new_output.width = static_cast<int>(input.width * scale + .5f);
617 new_output.height = static_cast<int>(input.height * scale + .5f);
618 int new_pixels = new_output.width * new_output.height;
619 int num_pixels = GetOutputNumPixels();
620 return new_pixels <= num_pixels;
621}
622
623// Called by all coordinators when there is a change.
624bool CoordinatedVideoAdapter::AdaptToMinimumFormat(int* new_width,
625 int* new_height) {
626 VideoFormat new_output = output_format();
627 VideoFormat input = input_format();
628 if (input_format().IsSize0x0()) {
629 input = new_output;
630 }
631 int old_num_pixels = GetOutputNumPixels();
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000632 int min_num_pixels = INT_MAX;
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000633 adapt_reason_ = ADAPTREASON_NONE;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000634
635 // Reduce resolution based on encoder bandwidth (GD).
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000636 if (encoder_desired_num_pixels_ &&
637 (encoder_desired_num_pixels_ < min_num_pixels)) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000638 adapt_reason_ |= ADAPTREASON_BANDWIDTH;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000639 min_num_pixels = encoder_desired_num_pixels_;
640 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000641 // Reduce resolution based on CPU.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000642 if (cpu_adaptation_ && cpu_desired_num_pixels_ &&
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000643 (cpu_desired_num_pixels_ <= min_num_pixels)) {
644 if (cpu_desired_num_pixels_ < min_num_pixels) {
645 adapt_reason_ = ADAPTREASON_CPU;
646 } else {
647 adapt_reason_ |= ADAPTREASON_CPU;
648 }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000649 min_num_pixels = cpu_desired_num_pixels_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000650 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000651 // Round resolution for GD or CPU to allow 1/2 to map to 9/16.
652 if (!input.IsSize0x0() && min_num_pixels != INT_MAX) {
653 float scale = FindClosestScale(input.width, input.height, min_num_pixels);
654 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
655 static_cast<int>(input.height * scale + .5f);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000656 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000657 // Reduce resolution based on View Request.
658 if (view_desired_num_pixels_ <= min_num_pixels) {
659 if (view_desired_num_pixels_ < min_num_pixels) {
660 adapt_reason_ = ADAPTREASON_VIEW;
661 } else {
662 adapt_reason_ |= ADAPTREASON_VIEW;
663 }
664 min_num_pixels = view_desired_num_pixels_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000665 }
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000666 // Snap to a scale factor.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000667 float scale = 1.0f;
668 if (!input.IsSize0x0()) {
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000669 scale = FindLowerScale(input.width, input.height, min_num_pixels);
670 min_num_pixels = static_cast<int>(input.width * scale + .5f) *
671 static_cast<int>(input.height * scale + .5f);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000672 }
673 if (scale == 1.0f) {
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000674 adapt_reason_ = ADAPTREASON_NONE;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000675 }
676 *new_width = new_output.width = static_cast<int>(input.width * scale + .5f);
677 *new_height = new_output.height = static_cast<int>(input.height * scale +
678 .5f);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000679 SetOutputNumPixels(min_num_pixels);
680
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000681 new_output.interval = view_desired_interval_;
682 SetOutputFormat(new_output);
683 int new_num_pixels = GetOutputNumPixels();
684 bool changed = new_num_pixels != old_num_pixels;
685
686 static const char* kReasons[8] = {
687 "None",
688 "CPU",
689 "BANDWIDTH",
690 "CPU+BANDWIDTH",
691 "VIEW",
692 "CPU+VIEW",
693 "BANDWIDTH+VIEW",
694 "CPU+BANDWIDTH+VIEW",
695 };
696
697 LOG(LS_VERBOSE) << "VAdapt Status View: " << view_desired_num_pixels_
698 << " GD: " << encoder_desired_num_pixels_
699 << " CPU: " << cpu_desired_num_pixels_
700 << " Pixels: " << min_num_pixels
701 << " Input: " << input.width
702 << "x" << input.height
703 << " Scale: " << scale
704 << " Resolution: " << new_output.width
705 << "x" << new_output.height
706 << " Changed: " << (changed ? "true" : "false")
707 << " Reason: " << kReasons[adapt_reason_];
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000708
709 if (changed) {
710 // When any adaptation occurs, historic CPU load levels are no longer
711 // accurate. Clear out our state so we can re-learn at the new normal.
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000712 cpu_load_num_samples_ = 0;
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000713 system_load_average_ = kCpuLoadInitialAverage;
714 }
715
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000716 return changed;
717}
718
719} // namespace cricket