This cl do a major cleanup of the VideoAdapter and make sure it does care about the VideoSinkWants.max_pixel_count and VideoSinkWants.max_pixel_count_step_up.
Unit tests are updated to test that screen share is not adapted but it does not change the VideoSinkWants in WebRtcVideoEngine2::SendStream due to a switch to screen share. The reason is that it works anyway and sprang is looking into how to do adaptation based on frame rate as well and use the adapter for screen share as well.
BUG=webrtc:5688, webrtc:5426
R=nisse@webrtc.org, pbos@webrtc.org, sprang@google.com
Review URL: https://codereview.webrtc.org/1836043004 .
Cr-Commit-Position: refs/heads/master@{#12240}
diff --git a/webrtc/media/base/videoadapter.h b/webrtc/media/base/videoadapter.h
index 13a342d..b7aba14 100644
--- a/webrtc/media/base/videoadapter.h
+++ b/webrtc/media/base/videoadapter.h
@@ -8,188 +8,72 @@
* be found in the AUTHORS file in the root of the source tree.
*/
-#ifndef WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
+#ifndef WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_
#define WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_
-#include "webrtc/base/common.h" // For ASSERT
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/optional.h"
-#include "webrtc/base/sigslot.h"
#include "webrtc/media/base/videocommon.h"
namespace cricket {
-class VideoFrame;
-
// VideoAdapter adapts an input video frame to an output frame based on the
// specified input and output formats. The adaptation includes dropping frames
-// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
+// to reduce frame rate and scaling frames.
+// VideoAdapter is thread safe.
class VideoAdapter {
public:
VideoAdapter();
virtual ~VideoAdapter();
- virtual void SetInputFormat(const VideoFormat& format);
- void SetOutputFormat(const VideoFormat& format);
- // Constrain output resolution to this many pixels overall
- void SetOutputNumPixels(int num_pixels);
- int GetOutputNumPixels() const;
-
- const VideoFormat& input_format();
- // Returns true if the adapter will always return zero size from
- // AdaptFrameResolution.
- bool drops_all_frames() const;
- const VideoFormat& output_format();
+ // Sets the expected frame interval. This controls how often frames should
+ // be dropped if |OnOutputFormatRequest| is called with a lower frame
+ // interval.
+ void SetExpectedInputFrameInterval(int64_t interval);
// Return the adapted resolution given the input resolution. The returned
// resolution will be 0x0 if the frame should be dropped.
VideoFormat AdaptFrameResolution(int in_width, int in_height);
- void set_scale_third(bool enable);
- bool scale_third() const { return scale_third_; }
+ // Requests the output frame size and frame interval from
+ // |AdaptFrameResolution| to not be larger than |format|.
+ void OnOutputFormatRequest(const VideoFormat& format);
- int adaptation_changes() const { return adaption_changes_; }
+ // Requests the output frame size from |AdaptFrameResolution| to not have
+ // more than |max_pixel_count| pixels and have "one step" up more pixels than
+ // max_pixel_count_step_up.
+ void OnResolutionRequest(rtc::Optional<int> max_pixel_count,
+ rtc::Optional<int> max_pixel_count_step_up);
- protected:
- float FindClosestScale(int width, int height, int target_num_pixels);
- float FindClosestViewScale(int width, int height, int target_num_pixels);
- float FindLowerScale(int width, int height, int target_num_pixels);
+ const VideoFormat& input_format() const;
private:
- const float* GetViewScaleFactors() const;
- float FindScale(const float* scale_factors,
- const float upbias, int width, int height,
- int target_num_pixels);
+ void SetInputFormat(const VideoFormat& format);
+ bool Adapt(int max_num_pixels, int max_pixel_count_step_up);
VideoFormat input_format_;
VideoFormat output_format_;
int output_num_pixels_;
- bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3.
- int frames_in_; // Number of input frames.
- int frames_out_; // Number of output frames.
- int frames_scaled_; // Number of frames scaled.
+ int frames_in_; // Number of input frames.
+ int frames_out_; // Number of output frames.
+ int frames_scaled_; // Number of frames scaled.
int adaption_changes_; // Number of changes in scale factor.
- size_t previous_width_; // Previous adapter output width.
- size_t previous_height_; // Previous adapter output height.
+ int previous_width_; // Previous adapter output width.
+ int previous_height_; // Previous adapter output height.
int64_t interval_next_frame_;
+
+ // Max number of pixels requested via calls to OnOutputFormatRequest,
+ // OnResolutionRequest respectively.
+ // The adapted output format is the minimum of these.
+ int format_request_max_pixel_count_;
+ int resolution_request_max_pixel_count_;
+
// The critical section to protect the above variables.
rtc::CriticalSection critical_section_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
};
-// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
-// the format request from the server, the resolution request from the encoder,
-// and the CPU load.
-class CoordinatedVideoAdapter
- : public VideoAdapter, public sigslot::has_slots<> {
- public:
- enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
- enum AdaptReasonEnum {
- ADAPTREASON_NONE = 0,
- ADAPTREASON_CPU = 1,
- ADAPTREASON_BANDWIDTH = 2,
- ADAPTREASON_VIEW = 4
- };
- typedef int AdaptReason;
-
- CoordinatedVideoAdapter();
- virtual ~CoordinatedVideoAdapter() {}
-
- virtual void SetInputFormat(const VideoFormat& format);
-
- // Enable or disable video adaptation due to the change of the CPU load.
- void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
- bool cpu_adaptation() const { return cpu_adaptation_; }
- // Enable or disable smoothing when doing CPU adaptation. When smoothing is
- // enabled, system CPU load is tracked using an exponential weighted
- // average.
- void set_cpu_smoothing(bool enable);
- bool cpu_smoothing() const { return cpu_smoothing_; }
- // Enable or disable video adaptation due to the change of the GD
- void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
- bool gd_adaptation() const { return gd_adaptation_; }
- // Enable or disable video adaptation due to the change of the View
- void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
- bool view_adaptation() const { return view_adaptation_; }
- // Enable or disable video adaptation to fast switch View
- void set_view_switch(bool enable) { view_switch_ = enable; }
- bool view_switch() const { return view_switch_; }
-
- CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
- return adapt_reason_;
- }
-
- // When the video is decreased, set the waiting time for CPU adaptation to
- // decrease video again.
- void set_cpu_load_min_samples(int cpu_load_min_samples);
- int cpu_load_min_samples() const { return cpu_load_min_samples_; }
- // CPU system load high threshold for reducing resolution. e.g. 0.85f
- void set_high_system_threshold(float high_system_threshold);
- float high_system_threshold() const { return high_system_threshold_; }
- // CPU system load low threshold for increasing resolution. e.g. 0.70f
- void set_low_system_threshold(float low_system_threshold);
- float low_system_threshold() const { return low_system_threshold_; }
- // CPU process load threshold for reducing resolution. e.g. 0.10f
- void set_process_threshold(float process_threshold);
- float process_threshold() const { return process_threshold_; }
-
- // Handle the format request from the server via Jingle update message.
- void OnOutputFormatRequest(const VideoFormat& format);
- // Handle the resolution request from the encoder due to bandwidth changes.
- void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
- // Handle the resolution request for CPU overuse.
- void OnCpuResolutionRequest(AdaptRequest request);
- void OnCpuResolutionRequest(rtc::Optional<int> max_pixel_count,
- rtc::Optional<int> max_pixel_count_step_up);
-
- // Handle the CPU load provided by a CPU monitor.
- void OnCpuLoadUpdated(int current_cpus, int max_cpus,
- float process_load, float system_load);
-
- sigslot::signal0<> SignalCpuAdaptationUnable;
-
- private:
- // Adapt to the minimum of the formats the server requests, the CPU wants, and
- // the encoder wants. Returns true if resolution changed.
- bool AdaptToMinimumFormat(int* new_width, int* new_height);
- bool IsMinimumFormat(int pixels);
- void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
- int* num_pixels);
- CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
- int current_cpus, int max_cpus,
- float process_load, float system_load);
-
- bool cpu_adaptation_; // True if cpu adaptation is enabled.
- bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation).
- bool gd_adaptation_; // True if gd adaptation is enabled.
- bool view_adaptation_; // True if view adaptation is enabled.
- bool view_switch_; // True if view switch is enabled.
- int cpu_downgrade_count_;
- int cpu_load_min_samples_;
- int cpu_load_num_samples_;
- // cpu system load thresholds relative to max cpus.
- float high_system_threshold_;
- float low_system_threshold_;
- // cpu process load thresholds relative to current cpus.
- float process_threshold_;
- // Video formats that the server view requests, the CPU wants, and the encoder
- // wants respectively. The adapted output format is the minimum of these.
- int view_desired_num_pixels_;
- int64_t view_desired_interval_;
- int encoder_desired_num_pixels_;
- int cpu_desired_num_pixels_;
- CoordinatedVideoAdapter::AdaptReason adapt_reason_;
- // The critical section to protect handling requests.
- rtc::CriticalSection request_critical_section_;
-
- // The weighted average of cpu load over time. It's always updated (if cpu
- // adaptation is on), but only used if cpu_smoothing_ is set.
- float system_load_average_;
-
- RTC_DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
-};
-
} // namespace cricket
-#endif // WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
+#endif // WEBRTC_MEDIA_BASE_VIDEOADAPTER_H_