blob: 9aab3d259cc23a3c143e0031ffebb26f96bb335a [file] [log] [blame]
henrike@webrtc.org28e20752013-07-10 00:45:36 +00001// libjingle
2// Copyright 2010 Google Inc.
3//
4// Redistribution and use in source and binary forms, with or without
5// modification, are permitted provided that the following conditions are met:
6//
7// 1. Redistributions of source code must retain the above copyright notice,
8// this list of conditions and the following disclaimer.
9// 2. Redistributions in binary form must reproduce the above copyright notice,
10// this list of conditions and the following disclaimer in the documentation
11// and/or other materials provided with the distribution.
12// 3. The name of the author may not be used to endorse or promote products
13// derived from this software without specific prior written permission.
14//
15// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
16// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
17// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
18// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
19// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
20// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
21// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
22// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
23// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
24// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26#ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
27#define TALK_MEDIA_BASE_VIDEOADAPTER_H_
28
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000029#include "talk/media/base/videocommon.h"
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +000030#include "webrtc/base/common.h" // For ASSERT
31#include "webrtc/base/criticalsection.h"
32#include "webrtc/base/scoped_ptr.h"
33#include "webrtc/base/sigslot.h"
henrike@webrtc.org28e20752013-07-10 00:45:36 +000034
35namespace cricket {
36
37class VideoFrame;
38
39// VideoAdapter adapts an input video frame to an output frame based on the
40// specified input and output formats. The adaptation includes dropping frames
41// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
42class VideoAdapter {
43 public:
44 VideoAdapter();
45 virtual ~VideoAdapter();
46
mallinath@webrtc.org1b15f422013-09-06 22:56:28 +000047 virtual void SetInputFormat(const VideoFormat& format);
henrike@webrtc.org28e20752013-07-10 00:45:36 +000048 void SetOutputFormat(const VideoFormat& format);
49 // Constrain output resolution to this many pixels overall
50 void SetOutputNumPixels(int num_pixels);
51 int GetOutputNumPixels() const;
52
53 const VideoFormat& input_format();
henrike@webrtc.org704bf9e2014-02-27 17:52:04 +000054 // Returns true if the adapter is dropping frames in calls to AdaptFrame.
55 bool drops_all_frames() const;
henrike@webrtc.org28e20752013-07-10 00:45:36 +000056 const VideoFormat& output_format();
57 // If the parameter black is true, the adapted frames will be black.
58 void SetBlackOutput(bool black);
magjed@webrtc.orgf58b4552014-11-19 18:09:14 +000059 bool IsBlackOutput();
60
61 // Return the adapted resolution given the input resolution. The returned
62 // resolution will be 0x0 if the frame should be dropped.
63 VideoFormat AdaptFrameResolution(int in_width, int in_height);
henrike@webrtc.org28e20752013-07-10 00:45:36 +000064
65 // Adapt the input frame from the input format to the output format. Return
66 // true and set the output frame to NULL if the input frame is dropped. Return
67 // true and set the out frame to output_frame_ if the input frame is adapted
68 // successfully. Return false otherwise.
buildbot@webrtc.org0cdcd232014-06-04 01:31:14 +000069 // Note that, if no adaptation is required, |out_frame| will refer directly
70 // in_frame. If a copy is always required, the caller must do an explicit
71 // copy.
72 // If a copy has taken place, |output_frame_| is owned by the VideoAdapter
73 // and will remain usable until the adapter is destroyed or AdaptFrame is
74 // called again.
75 bool AdaptFrame(VideoFrame* in_frame, VideoFrame** out_frame);
henrike@webrtc.org28e20752013-07-10 00:45:36 +000076
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +000077 void set_scale_third(bool enable);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000078 bool scale_third() const { return scale_third_; }
79
buildbot@webrtc.org71dffb72014-06-24 07:24:49 +000080 int adaptation_changes() const { return adaption_changes_; }
81
henrike@webrtc.org28e20752013-07-10 00:45:36 +000082 protected:
83 float FindClosestScale(int width, int height, int target_num_pixels);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000084 float FindClosestViewScale(int width, int height, int target_num_pixels);
henrike@webrtc.org28e20752013-07-10 00:45:36 +000085 float FindLowerScale(int width, int height, int target_num_pixels);
86
87 private:
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000088 const float* GetViewScaleFactors() const;
89 float FindScale(const float* scale_factors,
90 const float upbias, int width, int height,
91 int target_num_pixels);
henrike@webrtc.org28e20752013-07-10 00:45:36 +000092 bool StretchToOutputFrame(const VideoFrame* in_frame);
93
94 VideoFormat input_format_;
95 VideoFormat output_format_;
96 int output_num_pixels_;
wu@webrtc.orgcadf9042013-08-30 21:24:16 +000097 bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3.
sergeyu@chromium.org9cf037b2014-02-07 19:03:26 +000098 int frames_in_; // Number of input frames.
99 int frames_out_; // Number of output frames.
100 int frames_scaled_; // Number of frames scaled.
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000101 int adaption_changes_; // Number of changes in scale factor.
magjed@webrtc.orga73d7462014-11-14 13:25:25 +0000102 size_t previous_width_; // Previous adapter output width.
103 size_t previous_height_; // Previous adapter output height.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000104 bool black_output_; // Flag to tell if we need to black output_frame_.
105 bool is_black_; // Flag to tell if output_frame_ is currently black.
106 int64 interval_next_frame_;
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000107 rtc::scoped_ptr<VideoFrame> output_frame_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000108 // The critical section to protect the above variables.
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000109 rtc::CriticalSection critical_section_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000110
111 DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
112};
113
114// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
115// the format request from the server, the resolution request from the encoder,
116// and the CPU load.
117class CoordinatedVideoAdapter
118 : public VideoAdapter, public sigslot::has_slots<> {
119 public:
120 enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
wu@webrtc.org364f2042013-11-20 21:49:41 +0000121 enum AdaptReasonEnum {
henrike@webrtc.orga7b98182014-02-21 15:51:43 +0000122 ADAPTREASON_NONE = 0,
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000123 ADAPTREASON_CPU = 1,
124 ADAPTREASON_BANDWIDTH = 2,
125 ADAPTREASON_VIEW = 4
126 };
127 typedef int AdaptReason;
128
129 CoordinatedVideoAdapter();
130 virtual ~CoordinatedVideoAdapter() {}
131
mallinath@webrtc.org1b15f422013-09-06 22:56:28 +0000132 virtual void SetInputFormat(const VideoFormat& format);
133
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000134 // Enable or disable video adaptation due to the change of the CPU load.
135 void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
136 bool cpu_adaptation() const { return cpu_adaptation_; }
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000137 // Enable or disable smoothing when doing CPU adaptation. When smoothing is
138 // enabled, system CPU load is tracked using an exponential weighted
139 // average.
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000140 void set_cpu_smoothing(bool enable);
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000141 bool cpu_smoothing() const { return cpu_smoothing_; }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000142 // Enable or disable video adaptation due to the change of the GD
143 void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
144 bool gd_adaptation() const { return gd_adaptation_; }
145 // Enable or disable video adaptation due to the change of the View
146 void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
147 bool view_adaptation() const { return view_adaptation_; }
148 // Enable or disable video adaptation to fast switch View
149 void set_view_switch(bool enable) { view_switch_ = enable; }
150 bool view_switch() const { return view_switch_; }
151
152 CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
153 return adapt_reason_;
154 }
155
156 // When the video is decreased, set the waiting time for CPU adaptation to
157 // decrease video again.
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000158 void set_cpu_load_min_samples(int cpu_load_min_samples);
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000159 int cpu_load_min_samples() const { return cpu_load_min_samples_; }
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000160 // CPU system load high threshold for reducing resolution. e.g. 0.85f
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000161 void set_high_system_threshold(float high_system_threshold);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000162 float high_system_threshold() const { return high_system_threshold_; }
163 // CPU system load low threshold for increasing resolution. e.g. 0.70f
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000164 void set_low_system_threshold(float low_system_threshold);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000165 float low_system_threshold() const { return low_system_threshold_; }
166 // CPU process load threshold for reducing resolution. e.g. 0.10f
henrike@webrtc.orgd43aa9d2014-02-21 23:43:24 +0000167 void set_process_threshold(float process_threshold);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000168 float process_threshold() const { return process_threshold_; }
169
170 // Handle the format request from the server via Jingle update message.
171 void OnOutputFormatRequest(const VideoFormat& format);
172 // Handle the resolution request from the encoder due to bandwidth changes.
173 void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
wu@webrtc.orgcadf9042013-08-30 21:24:16 +0000174 // Handle the resolution request for CPU overuse.
175 void OnCpuResolutionRequest(AdaptRequest request);
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000176 // Handle the CPU load provided by a CPU monitor.
177 void OnCpuLoadUpdated(int current_cpus, int max_cpus,
178 float process_load, float system_load);
179
180 sigslot::signal0<> SignalCpuAdaptationUnable;
181
182 private:
183 // Adapt to the minimum of the formats the server requests, the CPU wants, and
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000184 // the encoder wants. Returns true if resolution changed.
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000185 bool AdaptToMinimumFormat(int* new_width, int* new_height);
186 bool IsMinimumFormat(int pixels);
187 void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
188 int* num_pixels);
189 CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
190 int current_cpus, int max_cpus,
191 float process_load, float system_load);
192
193 bool cpu_adaptation_; // True if cpu adaptation is enabled.
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000194 bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation).
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000195 bool gd_adaptation_; // True if gd adaptation is enabled.
196 bool view_adaptation_; // True if view adaptation is enabled.
197 bool view_switch_; // True if view switch is enabled.
198 int cpu_downgrade_count_;
mallinath@webrtc.org67ee6b92014-02-03 16:57:16 +0000199 int cpu_load_min_samples_;
200 int cpu_load_num_samples_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000201 // cpu system load thresholds relative to max cpus.
202 float high_system_threshold_;
203 float low_system_threshold_;
204 // cpu process load thresholds relative to current cpus.
205 float process_threshold_;
206 // Video formats that the server view requests, the CPU wants, and the encoder
207 // wants respectively. The adapted output format is the minimum of these.
208 int view_desired_num_pixels_;
209 int64 view_desired_interval_;
210 int encoder_desired_num_pixels_;
211 int cpu_desired_num_pixels_;
212 CoordinatedVideoAdapter::AdaptReason adapt_reason_;
213 // The critical section to protect handling requests.
buildbot@webrtc.orgd4e598d2014-07-29 17:36:52 +0000214 rtc::CriticalSection request_critical_section_;
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000215
henrike@webrtc.org28654cb2013-07-22 21:07:49 +0000216 // The weighted average of cpu load over time. It's always updated (if cpu
217 // adaptation is on), but only used if cpu_smoothing_ is set.
218 float system_load_average_;
219
henrike@webrtc.org28e20752013-07-10 00:45:36 +0000220 DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
221};
222
223} // namespace cricket
224
225#endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT