blob: b247004ec282c7c262958f89157507838ba2d4eb [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Per512ecb32016-09-23 15:52:06 +020015#include <utility>
16
Jiawei Ouc2ebe212018-11-08 10:02:56 -080017#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020018#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010019#include "api/video/video_bitrate_allocation.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020020#include "api/video_codecs/create_vp8_temporal_layers.h"
21#include "api/video_codecs/vp8_temporal_layers.h"
Steve Anton10542f22019-01-11 09:11:00 -080022#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020023#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080025#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080027#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010028#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020029#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/sleep.h"
31#include "test/encoder_settings.h"
32#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020033#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "test/frame_generator.h"
35#include "test/gmock.h"
36#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020037#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070039
40namespace webrtc {
41
sprangb1ca0732017-02-01 08:38:12 -080042using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080043using ::testing::_;
44using ::testing::Return;
kthelgason876222f2016-11-29 01:44:11 -080045
perkj803d97f2016-11-01 11:45:46 -070046namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020047const int kMinPixelsPerFrame = 320 * 180;
48const int kMinFramerateFps = 2;
49const int kMinBalancedFramerateFps = 7;
50const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080051const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010052const uint32_t kTargetBitrateBps = 1000000;
53const uint32_t kSimulcastTargetBitrateBps = 3150000;
54const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080055const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070056const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020057const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080058
perkj803d97f2016-11-01 11:45:46 -070059class TestBuffer : public webrtc::I420Buffer {
60 public:
61 TestBuffer(rtc::Event* event, int width, int height)
62 : I420Buffer(width, height), event_(event) {}
63
64 private:
65 friend class rtc::RefCountedObject<TestBuffer>;
66 ~TestBuffer() override {
67 if (event_)
68 event_->Set();
69 }
70 rtc::Event* const event_;
71};
72
Niels Möller7dc26b72017-12-06 10:27:48 +010073class CpuOveruseDetectorProxy : public OveruseFrameDetector {
74 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020075 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
76 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010077 last_target_framerate_fps_(-1) {}
78 virtual ~CpuOveruseDetectorProxy() {}
79
80 void OnTargetFramerateUpdated(int framerate_fps) override {
81 rtc::CritScope cs(&lock_);
82 last_target_framerate_fps_ = framerate_fps;
83 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
84 }
85
86 int GetLastTargetFramerate() {
87 rtc::CritScope cs(&lock_);
88 return last_target_framerate_fps_;
89 }
90
Niels Möller4db138e2018-04-19 09:04:13 +020091 CpuOveruseOptions GetOptions() { return options_; }
92
Niels Möller7dc26b72017-12-06 10:27:48 +010093 private:
94 rtc::CriticalSection lock_;
95 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
96};
97
mflodmancc3d4422017-08-03 08:27:51 -070098class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -070099 public:
Niels Möller213618e2018-07-24 09:29:58 +0200100 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
101 const VideoStreamEncoderSettings& settings)
Yves Gerey665174f2018-06-19 15:03:05 +0200102 : VideoStreamEncoder(1 /* number_of_cores */,
103 stats_proxy,
104 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200105 std::unique_ptr<OveruseFrameDetector>(
106 overuse_detector_proxy_ =
107 new CpuOveruseDetectorProxy(stats_proxy))) {}
perkj803d97f2016-11-01 11:45:46 -0700108
sprangb1ca0732017-02-01 08:38:12 -0800109 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100110 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800111 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800112 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700113 event.Set();
114 });
perkj070ba852017-02-16 15:46:27 -0800115 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700116 }
117
kthelgason2fc52542017-03-03 00:24:41 -0800118 // This is used as a synchronisation mechanism, to make sure that the
119 // encoder queue is not blocked before we start sending it frames.
120 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100121 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200122 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800123 ASSERT_TRUE(event.Wait(5000));
124 }
125
sprangb1ca0732017-02-01 08:38:12 -0800126 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800127
sprangb1ca0732017-02-01 08:38:12 -0800128 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800129
sprangb1ca0732017-02-01 08:38:12 -0800130 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800131
sprangb1ca0732017-02-01 08:38:12 -0800132 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700133
Niels Möller7dc26b72017-12-06 10:27:48 +0100134 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700135};
136
asapersson5f7226f2016-11-25 04:37:00 -0800137class VideoStreamFactory
138 : public VideoEncoderConfig::VideoStreamFactoryInterface {
139 public:
sprangfda496a2017-06-15 04:21:07 -0700140 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
141 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800142 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700143 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800144 }
145
146 private:
147 std::vector<VideoStream> CreateEncoderStreams(
148 int width,
149 int height,
150 const VideoEncoderConfig& encoder_config) override {
151 std::vector<VideoStream> streams =
152 test::CreateVideoStreams(width, height, encoder_config);
153 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100154 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700155 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800156 }
157 return streams;
158 }
sprangfda496a2017-06-15 04:21:07 -0700159
asapersson5f7226f2016-11-25 04:37:00 -0800160 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700161 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800162};
163
sprangb1ca0732017-02-01 08:38:12 -0800164class AdaptingFrameForwarder : public test::FrameForwarder {
165 public:
166 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700167 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800168
169 void set_adaptation_enabled(bool enabled) {
170 rtc::CritScope cs(&crit_);
171 adaptation_enabled_ = enabled;
172 }
173
asaperssonfab67072017-04-04 05:51:49 -0700174 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800175 rtc::CritScope cs(&crit_);
176 return adaptation_enabled_;
177 }
178
asapersson09f05612017-05-15 23:40:18 -0700179 rtc::VideoSinkWants last_wants() const {
180 rtc::CritScope cs(&crit_);
181 return last_wants_;
182 }
183
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200184 absl::optional<int> last_sent_width() const { return last_width_; }
185 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800186
sprangb1ca0732017-02-01 08:38:12 -0800187 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
188 int cropped_width = 0;
189 int cropped_height = 0;
190 int out_width = 0;
191 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700192 if (adaption_enabled()) {
193 if (adapter_.AdaptFrameResolution(
194 video_frame.width(), video_frame.height(),
195 video_frame.timestamp_us() * 1000, &cropped_width,
196 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100197 VideoFrame adapted_frame =
198 VideoFrame::Builder()
199 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
200 nullptr, out_width, out_height))
201 .set_timestamp_rtp(99)
202 .set_timestamp_ms(99)
203 .set_rotation(kVideoRotation_0)
204 .build();
sprangc5d62e22017-04-02 23:53:04 -0700205 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
206 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800207 last_width_.emplace(adapted_frame.width());
208 last_height_.emplace(adapted_frame.height());
209 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200210 last_width_ = absl::nullopt;
211 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700212 }
sprangb1ca0732017-02-01 08:38:12 -0800213 } else {
214 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800215 last_width_.emplace(video_frame.width());
216 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800217 }
218 }
219
220 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
221 const rtc::VideoSinkWants& wants) override {
222 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700223 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700224 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
225 wants.max_pixel_count,
226 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800227 test::FrameForwarder::AddOrUpdateSink(sink, wants);
228 }
sprangb1ca0732017-02-01 08:38:12 -0800229 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700230 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
231 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200232 absl::optional<int> last_width_;
233 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800234};
sprangc5d62e22017-04-02 23:53:04 -0700235
Niels Möller213618e2018-07-24 09:29:58 +0200236// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700237class MockableSendStatisticsProxy : public SendStatisticsProxy {
238 public:
239 MockableSendStatisticsProxy(Clock* clock,
240 const VideoSendStream::Config& config,
241 VideoEncoderConfig::ContentType content_type)
242 : SendStatisticsProxy(clock, config, content_type) {}
243
244 VideoSendStream::Stats GetStats() override {
245 rtc::CritScope cs(&lock_);
246 if (mock_stats_)
247 return *mock_stats_;
248 return SendStatisticsProxy::GetStats();
249 }
250
Niels Möller213618e2018-07-24 09:29:58 +0200251 int GetInputFrameRate() const override {
252 rtc::CritScope cs(&lock_);
253 if (mock_stats_)
254 return mock_stats_->input_frame_rate;
255 return SendStatisticsProxy::GetInputFrameRate();
256 }
sprangc5d62e22017-04-02 23:53:04 -0700257 void SetMockStats(const VideoSendStream::Stats& stats) {
258 rtc::CritScope cs(&lock_);
259 mock_stats_.emplace(stats);
260 }
261
262 void ResetMockStats() {
263 rtc::CritScope cs(&lock_);
264 mock_stats_.reset();
265 }
266
267 private:
268 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200269 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700270};
271
sprang4847ae62017-06-27 07:06:52 -0700272class MockBitrateObserver : public VideoBitrateAllocationObserver {
273 public:
Erik Språng566124a2018-04-23 12:32:22 +0200274 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700275};
276
perkj803d97f2016-11-01 11:45:46 -0700277} // namespace
278
mflodmancc3d4422017-08-03 08:27:51 -0700279class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700280 public:
281 static const int kDefaultTimeoutMs = 30 * 1000;
282
mflodmancc3d4422017-08-03 08:27:51 -0700283 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700284 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700285 codec_width_(320),
286 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200287 max_framerate_(kDefaultFramerate),
perkj26091b12016-09-01 01:17:40 -0700288 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200289 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800290 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700291 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700292 Clock::GetRealTimeClock(),
293 video_send_config_,
294 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700295 sink_(&fake_encoder_) {}
296
297 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700298 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700299 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200300 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800301 video_send_config_.encoder_settings.bitrate_allocator_factory =
302 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200303 video_send_config_.rtp.payload_name = "FAKE";
304 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700305
Per512ecb32016-09-23 15:52:06 +0200306 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200307 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700308 video_encoder_config.video_stream_factory =
309 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100310 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700311
312 // Framerate limit is specified by the VideoStreamFactory.
313 std::vector<VideoStream> streams =
314 video_encoder_config.video_stream_factory->CreateEncoderStreams(
315 codec_width_, codec_height_, video_encoder_config);
316 max_framerate_ = streams[0].max_framerate;
317 fake_clock_.SetTimeMicros(1234);
318
Niels Möllerf1338562018-04-26 09:51:47 +0200319 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800320 }
321
Niels Möllerf1338562018-04-26 09:51:47 +0200322 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700323 if (video_stream_encoder_)
324 video_stream_encoder_->Stop();
325 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
perkj803d97f2016-11-01 11:45:46 -0700326 stats_proxy_.get(), video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700327 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
328 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700329 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700330 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
331 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200332 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700333 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800334 }
335
336 void ResetEncoder(const std::string& payload_name,
337 size_t num_streams,
338 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700339 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700340 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200341 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800342
343 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200344 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800345 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100346 video_encoder_config.max_bitrate_bps =
347 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800348 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700349 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
350 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700351 video_encoder_config.content_type =
352 screenshare ? VideoEncoderConfig::ContentType::kScreen
353 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700354 if (payload_name == "VP9") {
355 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
356 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
357 video_encoder_config.encoder_specific_settings =
358 new rtc::RefCountedObject<
359 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
360 }
Niels Möllerf1338562018-04-26 09:51:47 +0200361 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700362 }
363
sprang57c2fff2017-01-16 06:24:02 -0800364 VideoFrame CreateFrame(int64_t ntp_time_ms,
365 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100366 VideoFrame frame =
367 VideoFrame::Builder()
368 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
369 destruction_event, codec_width_, codec_height_))
370 .set_timestamp_rtp(99)
371 .set_timestamp_ms(99)
372 .set_rotation(kVideoRotation_0)
373 .build();
sprang57c2fff2017-01-16 06:24:02 -0800374 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700375 return frame;
376 }
377
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100378 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
379 rtc::Event* destruction_event,
380 int offset_x) const {
381 VideoFrame frame =
382 VideoFrame::Builder()
383 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
384 destruction_event, codec_width_, codec_height_))
385 .set_timestamp_rtp(99)
386 .set_timestamp_ms(99)
387 .set_rotation(kVideoRotation_0)
388 .set_update_rect({offset_x, 0, 1, 1})
389 .build();
390 frame.set_ntp_time_ms(ntp_time_ms);
391 return frame;
392 }
393
sprang57c2fff2017-01-16 06:24:02 -0800394 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100395 VideoFrame frame =
396 VideoFrame::Builder()
397 .set_video_frame_buffer(
398 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
399 .set_timestamp_rtp(99)
400 .set_timestamp_ms(99)
401 .set_rotation(kVideoRotation_0)
402 .build();
sprang57c2fff2017-01-16 06:24:02 -0800403 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700404 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700405 return frame;
406 }
407
asapersson02465b82017-04-10 01:12:52 -0700408 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700409 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700410 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
411 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700412 }
413
asapersson09f05612017-05-15 23:40:18 -0700414 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
415 const rtc::VideoSinkWants& wants2) {
416 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
417 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
418 }
419
Åsa Persson8c1bf952018-09-13 10:42:19 +0200420 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
421 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
422 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
423 EXPECT_FALSE(wants.target_pixel_count);
424 }
425
asapersson09f05612017-05-15 23:40:18 -0700426 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
427 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200428 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700429 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
430 EXPECT_GT(wants1.max_pixel_count, 0);
431 }
432
433 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
434 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200435 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700436 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
437 }
438
asaperssonf7e294d2017-06-13 23:25:22 -0700439 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
440 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200441 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700442 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
443 }
444
445 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
446 const rtc::VideoSinkWants& wants2) {
447 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
448 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
449 }
450
451 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
452 const rtc::VideoSinkWants& wants2) {
453 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
454 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
455 }
456
457 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
458 const rtc::VideoSinkWants& wants2) {
459 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
460 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
461 EXPECT_GT(wants1.max_pixel_count, 0);
462 }
463
464 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
465 const rtc::VideoSinkWants& wants2) {
466 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
467 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
468 }
469
asapersson09f05612017-05-15 23:40:18 -0700470 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
471 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200472 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700473 EXPECT_LT(wants.max_pixel_count, pixel_count);
474 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700475 }
476
477 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
478 EXPECT_LT(wants.max_framerate_fps, fps);
479 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
480 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700481 }
482
asaperssonf7e294d2017-06-13 23:25:22 -0700483 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
484 int expected_fps) {
485 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
486 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
487 EXPECT_FALSE(wants.target_pixel_count);
488 }
489
Jonathan Yubc771b72017-12-08 17:04:29 -0800490 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
491 int last_frame_pixels) {
492 // Balanced mode should always scale FPS to the desired range before
493 // attempting to scale resolution.
494 int fps_limit = wants.max_framerate_fps;
495 if (last_frame_pixels <= 320 * 240) {
496 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
497 } else if (last_frame_pixels <= 480 * 270) {
498 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
499 } else if (last_frame_pixels <= 640 * 480) {
500 EXPECT_LE(15, fps_limit);
501 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200502 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800503 }
504 }
505
sprang4847ae62017-06-27 07:06:52 -0700506 void WaitForEncodedFrame(int64_t expected_ntp_time) {
507 sink_.WaitForEncodedFrame(expected_ntp_time);
508 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
509 }
510
511 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
512 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
513 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
514 return ok;
515 }
516
517 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
518 sink_.WaitForEncodedFrame(expected_width, expected_height);
519 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
520 }
521
522 void ExpectDroppedFrame() {
523 sink_.ExpectDroppedFrame();
524 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
525 }
526
527 bool WaitForFrame(int64_t timeout_ms) {
528 bool ok = sink_.WaitForFrame(timeout_ms);
529 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
530 return ok;
531 }
532
perkj26091b12016-09-01 01:17:40 -0700533 class TestEncoder : public test::FakeEncoder {
534 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100535 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700536
asaperssonfab67072017-04-04 05:51:49 -0700537 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800538 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700539 return config_;
540 }
541
542 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800543 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700544 block_next_encode_ = true;
545 }
546
Erik Språngaed30702018-11-05 12:57:17 +0100547 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800548 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100549 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100550 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100551 if (quality_scaling_) {
552 info.scaling_settings =
553 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
554 }
555 info.is_hardware_accelerated = is_hardware_accelerated_;
Erik Språngaed30702018-11-05 12:57:17 +0100556 }
557 return info;
kthelgason876222f2016-11-29 01:44:11 -0800558 }
559
Erik Språngb7cb7b52019-02-26 15:52:33 +0100560 int32_t RegisterEncodeCompleteCallback(
561 EncodedImageCallback* callback) override {
562 rtc::CritScope lock(&local_crit_sect_);
563 encoded_image_callback_ = callback;
564 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
565 }
566
perkjfa10b552016-10-02 23:45:26 -0700567 void ContinueEncode() { continue_encode_event_.Set(); }
568
569 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
570 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800571 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700572 EXPECT_EQ(timestamp_, timestamp);
573 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
574 }
575
kthelgason2fc52542017-03-03 00:24:41 -0800576 void SetQualityScaling(bool b) {
577 rtc::CritScope lock(&local_crit_sect_);
578 quality_scaling_ = b;
579 }
kthelgasonad9010c2017-02-14 00:46:51 -0800580
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100581 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
582 rtc::CritScope lock(&local_crit_sect_);
583 is_hardware_accelerated_ = is_hardware_accelerated;
584 }
585
sprangfe627f32017-03-29 08:24:59 -0700586 void ForceInitEncodeFailure(bool force_failure) {
587 rtc::CritScope lock(&local_crit_sect_);
588 force_init_encode_failed_ = force_failure;
589 }
590
Niels Möller6bb5ab92019-01-11 11:11:10 +0100591 void SimulateOvershoot(double rate_factor) {
592 rtc::CritScope lock(&local_crit_sect_);
593 rate_factor_ = rate_factor;
594 }
595
Erik Språngd7329ca2019-02-21 21:19:53 +0100596 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100597 rtc::CritScope lock(&local_crit_sect_);
598 return last_framerate_;
599 }
600
Erik Språngd7329ca2019-02-21 21:19:53 +0100601 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100602 rtc::CritScope lock(&local_crit_sect_);
603 return last_update_rect_;
604 }
605
Erik Språngd7329ca2019-02-21 21:19:53 +0100606 const std::vector<FrameType>& LastFrameTypes() const {
607 rtc::CritScope lock(&local_crit_sect_);
608 return last_frame_types_;
609 }
610
611 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
612 const std::vector<FrameType> frame_type = {keyframe ? kVideoFrameKey
613 : kVideoFrameDelta};
614 {
615 rtc::CritScope lock(&local_crit_sect_);
616 last_frame_types_ = frame_type;
617 }
618 FakeEncoder::Encode(input_image, nullptr, &frame_type);
619 }
620
Erik Språngb7cb7b52019-02-26 15:52:33 +0100621 void InjectEncodedImage(const EncodedImage& image) {
622 rtc::CritScope lock(&local_crit_sect_);
623 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
624 }
625
Erik Språngd7329ca2019-02-21 21:19:53 +0100626 void ExpectNullFrame() {
627 rtc::CritScope lock(&local_crit_sect_);
628 expect_null_frame_ = true;
629 }
630
631 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
632 auto allocation = last_bitrate_allocation_;
633 last_bitrate_allocation_.reset();
634 return allocation;
635 }
636
perkjfa10b552016-10-02 23:45:26 -0700637 private:
perkj26091b12016-09-01 01:17:40 -0700638 int32_t Encode(const VideoFrame& input_image,
639 const CodecSpecificInfo* codec_specific_info,
640 const std::vector<FrameType>* frame_types) override {
641 bool block_encode;
642 {
brandtre78d2662017-01-16 05:57:16 -0800643 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100644 if (expect_null_frame_) {
645 EXPECT_EQ(input_image.timestamp(), 0u);
646 EXPECT_EQ(input_image.width(), 1);
647 last_frame_types_ = *frame_types;
648 expect_null_frame_ = false;
649 } else {
650 EXPECT_GT(input_image.timestamp(), timestamp_);
651 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
652 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
653 }
perkj26091b12016-09-01 01:17:40 -0700654
655 timestamp_ = input_image.timestamp();
656 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700657 last_input_width_ = input_image.width();
658 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700659 block_encode = block_next_encode_;
660 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100661 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100662 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700663 }
664 int32_t result =
665 FakeEncoder::Encode(input_image, codec_specific_info, frame_types);
666 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700667 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700668 return result;
669 }
670
sprangfe627f32017-03-29 08:24:59 -0700671 int32_t InitEncode(const VideoCodec* config,
672 int32_t number_of_cores,
673 size_t max_payload_size) override {
674 int res =
675 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
676 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100677 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100678 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700679 // Simulate setting up temporal layers, in order to validate the life
680 // cycle of these objects.
681 int num_streams = std::max<int>(1, config->numberOfSimulcastStreams);
sprangfe627f32017-03-29 08:24:59 -0700682 for (int i = 0; i < num_streams; ++i) {
683 allocated_temporal_layers_.emplace_back(
Erik Språng4529fbc2018-10-12 10:30:31 +0200684 CreateVp8TemporalLayers(Vp8TemporalLayersType::kFixedPattern,
685 config->VP8().numberOfTemporalLayers));
sprangfe627f32017-03-29 08:24:59 -0700686 }
687 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100688 if (force_init_encode_failed_) {
689 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700690 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100691 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100692
Erik Språngb7cb7b52019-02-26 15:52:33 +0100693 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700694 return res;
695 }
696
Erik Språngb7cb7b52019-02-26 15:52:33 +0100697 int32_t Release() override {
698 rtc::CritScope lock(&local_crit_sect_);
699 EXPECT_NE(initialized_, EncoderState::kUninitialized);
700 initialized_ = EncoderState::kUninitialized;
701 return FakeEncoder::Release();
702 }
703
Niels Möller6bb5ab92019-01-11 11:11:10 +0100704 int32_t SetRateAllocation(const VideoBitrateAllocation& rate_allocation,
705 uint32_t framerate) {
706 rtc::CritScope lock(&local_crit_sect_);
707 VideoBitrateAllocation adjusted_rate_allocation;
708 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
709 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
710 if (rate_allocation.HasBitrate(si, ti)) {
711 adjusted_rate_allocation.SetBitrate(
712 si, ti,
713 static_cast<uint32_t>(rate_allocation.GetBitrate(si, ti) *
714 rate_factor_));
715 }
716 }
717 }
718 last_framerate_ = framerate;
Erik Språngd7329ca2019-02-21 21:19:53 +0100719 last_bitrate_allocation_ = rate_allocation;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100720 return FakeEncoder::SetRateAllocation(adjusted_rate_allocation,
721 framerate);
722 }
723
brandtre78d2662017-01-16 05:57:16 -0800724 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100725 enum class EncoderState {
726 kUninitialized,
727 kInitializationFailed,
728 kInitialized
729 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
730 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700731 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700732 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700733 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
734 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
735 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
736 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
737 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100738 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Erik Språng4529fbc2018-10-12 10:30:31 +0200739 std::vector<std::unique_ptr<Vp8TemporalLayers>> allocated_temporal_layers_
danilchapa37de392017-09-09 04:17:22 -0700740 RTC_GUARDED_BY(local_crit_sect_);
741 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100742 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
743 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100744 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100745 VideoFrame::UpdateRect last_update_rect_
746 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Erik Språngd7329ca2019-02-21 21:19:53 +0100747 std::vector<FrameType> last_frame_types_;
748 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100749 EncodedImageCallback* encoded_image_callback_
750 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
perkj26091b12016-09-01 01:17:40 -0700751 };
752
mflodmancc3d4422017-08-03 08:27:51 -0700753 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700754 public:
755 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100756 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700757
perkj26091b12016-09-01 01:17:40 -0700758 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700759 EXPECT_TRUE(
760 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
761 }
762
763 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
764 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700765 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700766 if (!encoded_frame_event_.Wait(timeout_ms))
767 return false;
perkj26091b12016-09-01 01:17:40 -0700768 {
769 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800770 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700771 }
772 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700773 return true;
perkj26091b12016-09-01 01:17:40 -0700774 }
775
sprangb1ca0732017-02-01 08:38:12 -0800776 void WaitForEncodedFrame(uint32_t expected_width,
777 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700778 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100779 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700780 }
781
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100782 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700783 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800784 uint32_t width = 0;
785 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800786 {
787 rtc::CritScope lock(&crit_);
788 width = last_width_;
789 height = last_height_;
790 }
791 EXPECT_EQ(expected_height, height);
792 EXPECT_EQ(expected_width, width);
793 }
794
kthelgason2fc52542017-03-03 00:24:41 -0800795 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800796
sprangc5d62e22017-04-02 23:53:04 -0700797 bool WaitForFrame(int64_t timeout_ms) {
798 return encoded_frame_event_.Wait(timeout_ms);
799 }
800
perkj26091b12016-09-01 01:17:40 -0700801 void SetExpectNoFrames() {
802 rtc::CritScope lock(&crit_);
803 expect_frames_ = false;
804 }
805
asaperssonfab67072017-04-04 05:51:49 -0700806 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200807 rtc::CritScope lock(&crit_);
808 return number_of_reconfigurations_;
809 }
810
asaperssonfab67072017-04-04 05:51:49 -0700811 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200812 rtc::CritScope lock(&crit_);
813 return min_transmit_bitrate_bps_;
814 }
815
Erik Språngd7329ca2019-02-21 21:19:53 +0100816 void SetNumExpectedLayers(size_t num_layers) {
817 rtc::CritScope lock(&crit_);
818 num_expected_layers_ = num_layers;
819 }
820
Erik Språngb7cb7b52019-02-26 15:52:33 +0100821 int64_t GetLastCaptureTimeMs() const {
822 rtc::CritScope lock(&crit_);
823 return last_capture_time_ms_;
824 }
825
perkj26091b12016-09-01 01:17:40 -0700826 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700827 Result OnEncodedImage(
828 const EncodedImage& encoded_image,
829 const CodecSpecificInfo* codec_specific_info,
830 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200831 rtc::CritScope lock(&crit_);
832 EXPECT_TRUE(expect_frames_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100833 uint32_t timestamp = encoded_image.Timestamp();
834 if (last_timestamp_ != timestamp) {
835 num_received_layers_ = 1;
836 } else {
837 ++num_received_layers_;
838 }
839 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100840 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -0800841 last_width_ = encoded_image._encodedWidth;
842 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +0100843 if (num_received_layers_ == num_expected_layers_) {
844 encoded_frame_event_.Set();
845 }
sprangb1ca0732017-02-01 08:38:12 -0800846 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200847 }
848
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100849 void OnEncoderConfigurationChanged(
850 std::vector<VideoStream> streams,
851 VideoEncoderConfig::ContentType content_type,
852 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200853 rtc::CriticalSection crit_;
854 ++number_of_reconfigurations_;
855 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
856 }
857
perkj26091b12016-09-01 01:17:40 -0700858 rtc::CriticalSection crit_;
859 TestEncoder* test_encoder_;
860 rtc::Event encoded_frame_event_;
sprangb1ca0732017-02-01 08:38:12 -0800861 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100862 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -0800863 uint32_t last_height_ = 0;
864 uint32_t last_width_ = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100865 size_t num_expected_layers_ = 1;
866 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700867 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200868 int number_of_reconfigurations_ = 0;
869 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700870 };
871
872 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100873 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200874 int codec_width_;
875 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700876 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -0700877 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200878 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800879 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700880 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700881 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800882 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700883 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700884 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700885};
886
mflodmancc3d4422017-08-03 08:27:51 -0700887TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
888 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100889 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700890 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700891 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700892 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700893 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700894}
895
mflodmancc3d4422017-08-03 08:27:51 -0700896TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700897 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100898 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200899 // The encoder will cache up to one frame for a short duration. Adding two
900 // frames means that the first frame will be dropped and the second frame will
901 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700902 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200903 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700904 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700905
mflodmancc3d4422017-08-03 08:27:51 -0700906 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700907
Sebastian Janssona3177052018-04-10 13:05:49 +0200908 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700909 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200910 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
911
912 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700913 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700914}
915
mflodmancc3d4422017-08-03 08:27:51 -0700916TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
917 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700918 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700919 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700920
mflodmancc3d4422017-08-03 08:27:51 -0700921 video_stream_encoder_->OnBitrateUpdated(0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +0200922 // The encoder will cache up to one frame for a short duration. Adding two
923 // frames means that the first frame will be dropped and the second frame will
924 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -0700925 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +0200926 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700927
mflodmancc3d4422017-08-03 08:27:51 -0700928 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -0700929 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +0200930 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
931 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -0700932 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700933}
934
mflodmancc3d4422017-08-03 08:27:51 -0700935TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
936 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700937 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700938 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700939
940 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -0700941 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -0700942
perkja49cbd32016-09-16 07:53:41 -0700943 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700944 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -0700945 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700946}
947
mflodmancc3d4422017-08-03 08:27:51 -0700948TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
949 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700950
perkja49cbd32016-09-16 07:53:41 -0700951 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700952 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700953
mflodmancc3d4422017-08-03 08:27:51 -0700954 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700955 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +0100956 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700957 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
958 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700959}
960
mflodmancc3d4422017-08-03 08:27:51 -0700961TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
962 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700963
964 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -0700965 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700966 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700967 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
968 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -0700969 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
970 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700971 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -0700972 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -0700973
mflodmancc3d4422017-08-03 08:27:51 -0700974 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700975}
976
mflodmancc3d4422017-08-03 08:27:51 -0700977TEST_F(VideoStreamEncoderTest,
978 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
979 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Per21d45d22016-10-30 21:37:57 +0100980 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200981
982 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200983 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700984 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +0100985 // The encoder will have been configured once when the first frame is
986 // received.
987 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200988
989 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200990 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +0200991 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -0700992 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200993 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +0200994
995 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200996 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700997 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +0100998 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -0700999 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001000
mflodmancc3d4422017-08-03 08:27:51 -07001001 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001002}
1003
mflodmancc3d4422017-08-03 08:27:51 -07001004TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
1005 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001006
1007 // Capture a frame and wait for it to synchronize with the encoder thread.
1008 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001009 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001010 // The encoder will have been configured once.
1011 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001012 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1013 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1014
1015 codec_width_ *= 2;
1016 codec_height_ *= 2;
1017 // Capture a frame with a higher resolution and wait for it to synchronize
1018 // with the encoder thread.
1019 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001020 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001021 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1022 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001023 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001024
mflodmancc3d4422017-08-03 08:27:51 -07001025 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001026}
1027
mflodmancc3d4422017-08-03 08:27:51 -07001028TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001029 EXPECT_TRUE(video_source_.has_sinks());
1030 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001031 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001032 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001033 EXPECT_FALSE(video_source_.has_sinks());
1034 EXPECT_TRUE(new_video_source.has_sinks());
1035
mflodmancc3d4422017-08-03 08:27:51 -07001036 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001037}
1038
mflodmancc3d4422017-08-03 08:27:51 -07001039TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001040 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001041 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001042 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001043 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001044}
1045
Jonathan Yubc771b72017-12-08 17:04:29 -08001046TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1047 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001048 const int kWidth = 1280;
1049 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001050
1051 // We rely on the automatic resolution adaptation, but we handle framerate
1052 // adaptation manually by mocking the stats proxy.
1053 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001054
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001055 // Enable BALANCED preference, no initial limitation.
Jonathan Yubc771b72017-12-08 17:04:29 -08001056 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001057 video_stream_encoder_->SetSource(&video_source_,
1058 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001059 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001060 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001061 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001062 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1063
Jonathan Yubc771b72017-12-08 17:04:29 -08001064 // Adapt down as far as possible.
1065 rtc::VideoSinkWants last_wants;
1066 int64_t t = 1;
1067 int loop_count = 0;
1068 do {
1069 ++loop_count;
1070 last_wants = video_source_.sink_wants();
1071
1072 // Simulate the framerate we've been asked to adapt to.
1073 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1074 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1075 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1076 mock_stats.input_frame_rate = fps;
1077 stats_proxy_->SetMockStats(mock_stats);
1078
1079 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1080 sink_.WaitForEncodedFrame(t);
1081 t += frame_interval_ms;
1082
mflodmancc3d4422017-08-03 08:27:51 -07001083 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001084 VerifyBalancedModeFpsRange(
1085 video_source_.sink_wants(),
1086 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1087 } while (video_source_.sink_wants().max_pixel_count <
1088 last_wants.max_pixel_count ||
1089 video_source_.sink_wants().max_framerate_fps <
1090 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001091
Jonathan Yubc771b72017-12-08 17:04:29 -08001092 // Verify that we've adapted all the way down.
1093 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001094 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001095 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1096 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001097 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001098 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1099 *video_source_.last_sent_height());
1100 EXPECT_EQ(kMinBalancedFramerateFps,
1101 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001102
Jonathan Yubc771b72017-12-08 17:04:29 -08001103 // Adapt back up the same number of times we adapted down.
1104 for (int i = 0; i < loop_count - 1; ++i) {
1105 last_wants = video_source_.sink_wants();
1106
1107 // Simulate the framerate we've been asked to adapt to.
1108 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1109 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1110 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1111 mock_stats.input_frame_rate = fps;
1112 stats_proxy_->SetMockStats(mock_stats);
1113
1114 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1115 sink_.WaitForEncodedFrame(t);
1116 t += frame_interval_ms;
1117
mflodmancc3d4422017-08-03 08:27:51 -07001118 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001119 VerifyBalancedModeFpsRange(
1120 video_source_.sink_wants(),
1121 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1122 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1123 last_wants.max_pixel_count ||
1124 video_source_.sink_wants().max_framerate_fps >
1125 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001126 }
1127
Åsa Persson8c1bf952018-09-13 10:42:19 +02001128 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001129 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001130 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001131 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1132 EXPECT_EQ((loop_count - 1) * 2,
1133 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001134
mflodmancc3d4422017-08-03 08:27:51 -07001135 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001136}
mflodmancc3d4422017-08-03 08:27:51 -07001137TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
1138 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001139 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001140
sprangc5d62e22017-04-02 23:53:04 -07001141 const int kFrameWidth = 1280;
1142 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001143
Åsa Persson8c1bf952018-09-13 10:42:19 +02001144 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001145
kthelgason5e13d412016-12-01 03:59:51 -08001146 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001147 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001148 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001149 frame_timestamp += kFrameIntervalMs;
1150
perkj803d97f2016-11-01 11:45:46 -07001151 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001152 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001153 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001154 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001155 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001156 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001157
asapersson0944a802017-04-07 00:57:58 -07001158 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001159 // wanted resolution.
1160 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1161 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1162 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001163 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001164
1165 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001166 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001168 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001169
sprangc5d62e22017-04-02 23:53:04 -07001170 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001171 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001172
sprangc5d62e22017-04-02 23:53:04 -07001173 // Force an input frame rate to be available, or the adaptation call won't
1174 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001175 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001176 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001177 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001178 stats_proxy_->SetMockStats(stats);
1179
mflodmancc3d4422017-08-03 08:27:51 -07001180 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001181 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001182 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001183 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001184 frame_timestamp += kFrameIntervalMs;
1185
1186 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001187 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001188 EXPECT_EQ(std::numeric_limits<int>::max(),
1189 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001190 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001191
asapersson02465b82017-04-10 01:12:52 -07001192 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001193 video_stream_encoder_->SetSource(&new_video_source,
1194 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001195 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001196
mflodmancc3d4422017-08-03 08:27:51 -07001197 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001198 new_video_source.IncomingCapturedFrame(
1199 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001200 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001201 frame_timestamp += kFrameIntervalMs;
1202
1203 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001204 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001205
1206 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001207 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001208 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001209 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1210 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001211 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001212 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001213
1214 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001215 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001216 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001217 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1218 EXPECT_EQ(std::numeric_limits<int>::max(),
1219 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001220 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001221
mflodmancc3d4422017-08-03 08:27:51 -07001222 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001223}
1224
mflodmancc3d4422017-08-03 08:27:51 -07001225TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
1226 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001227
asaperssonfab67072017-04-04 05:51:49 -07001228 const int kWidth = 1280;
1229 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001230 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001231 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001232 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1233 EXPECT_FALSE(stats.bw_limited_resolution);
1234 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1235
1236 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001237 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001238 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001239 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001240
1241 stats = stats_proxy_->GetStats();
1242 EXPECT_TRUE(stats.bw_limited_resolution);
1243 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1244
1245 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001246 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001247 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001248 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001249
1250 stats = stats_proxy_->GetStats();
1251 EXPECT_FALSE(stats.bw_limited_resolution);
1252 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1253 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1254
mflodmancc3d4422017-08-03 08:27:51 -07001255 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001256}
1257
mflodmancc3d4422017-08-03 08:27:51 -07001258TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
1259 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001260
1261 const int kWidth = 1280;
1262 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001263 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001264 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001265 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1266 EXPECT_FALSE(stats.cpu_limited_resolution);
1267 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1268
1269 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001270 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001271 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001272 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001273
1274 stats = stats_proxy_->GetStats();
1275 EXPECT_TRUE(stats.cpu_limited_resolution);
1276 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1277
1278 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001279 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001280 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001281 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001282
1283 stats = stats_proxy_->GetStats();
1284 EXPECT_FALSE(stats.cpu_limited_resolution);
1285 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001286 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001287
mflodmancc3d4422017-08-03 08:27:51 -07001288 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001289}
1290
mflodmancc3d4422017-08-03 08:27:51 -07001291TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
1292 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001293
asaperssonfab67072017-04-04 05:51:49 -07001294 const int kWidth = 1280;
1295 const int kHeight = 720;
1296 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001297 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001298 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001299 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001300 EXPECT_FALSE(stats.cpu_limited_resolution);
1301 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1302
asaperssonfab67072017-04-04 05:51:49 -07001303 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001304 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001305 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001306 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001307 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001308 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001309 EXPECT_TRUE(stats.cpu_limited_resolution);
1310 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1311
1312 // Set new source with adaptation still enabled.
1313 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001314 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001315 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001316
asaperssonfab67072017-04-04 05:51:49 -07001317 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001318 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001319 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001320 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001321 EXPECT_TRUE(stats.cpu_limited_resolution);
1322 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1323
1324 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001325 video_stream_encoder_->SetSource(&new_video_source,
1326 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001327
asaperssonfab67072017-04-04 05:51:49 -07001328 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001329 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001330 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001331 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001332 EXPECT_FALSE(stats.cpu_limited_resolution);
1333 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1334
1335 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001336 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001337 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001338
asaperssonfab67072017-04-04 05:51:49 -07001339 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001340 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001341 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001342 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001343 EXPECT_TRUE(stats.cpu_limited_resolution);
1344 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1345
asaperssonfab67072017-04-04 05:51:49 -07001346 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001347 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001348 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001349 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001350 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001351 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001352 EXPECT_FALSE(stats.cpu_limited_resolution);
1353 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001354 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001355
mflodmancc3d4422017-08-03 08:27:51 -07001356 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001357}
1358
mflodmancc3d4422017-08-03 08:27:51 -07001359TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
1360 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001361
asaperssonfab67072017-04-04 05:51:49 -07001362 const int kWidth = 1280;
1363 const int kHeight = 720;
1364 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001365 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001366 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001367 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001368 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001369 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001370
1371 // Set new source with adaptation still enabled.
1372 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001373 video_stream_encoder_->SetSource(&new_video_source,
1374 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001375
asaperssonfab67072017-04-04 05:51:49 -07001376 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001377 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001378 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001379 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001380 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001381 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001382
asaperssonfab67072017-04-04 05:51:49 -07001383 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001384 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001385 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001386 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001387 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001388 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001389 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001390 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001391
asaperssonfab67072017-04-04 05:51:49 -07001392 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001393 video_stream_encoder_->SetSource(&new_video_source,
1394 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001395
asaperssonfab67072017-04-04 05:51:49 -07001396 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001397 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001398 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001399 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001400 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001401 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001402
asapersson02465b82017-04-10 01:12:52 -07001403 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001405 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001406
asaperssonfab67072017-04-04 05:51:49 -07001407 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001408 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001409 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001410 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001411 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001412 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1413 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001414
mflodmancc3d4422017-08-03 08:27:51 -07001415 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001416}
1417
mflodmancc3d4422017-08-03 08:27:51 -07001418TEST_F(VideoStreamEncoderTest,
1419 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
1420 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001421
1422 const int kWidth = 1280;
1423 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001424 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001425 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001426 video_source_.IncomingCapturedFrame(
1427 CreateFrame(timestamp_ms, kWidth, kHeight));
1428 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001429 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1430 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1431 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1432
1433 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001434 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001435 timestamp_ms += kFrameIntervalMs;
1436 video_source_.IncomingCapturedFrame(
1437 CreateFrame(timestamp_ms, kWidth, kHeight));
1438 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001439 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1440 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1441 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1442
1443 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001444 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001445 timestamp_ms += kFrameIntervalMs;
1446 video_source_.IncomingCapturedFrame(
1447 CreateFrame(timestamp_ms, kWidth, kHeight));
1448 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001449 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1450 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1451 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1452
Niels Möller4db138e2018-04-19 09:04:13 +02001453 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001454 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001455
1456 VideoEncoderConfig video_encoder_config;
1457 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1458 // Make format different, to force recreation of encoder.
1459 video_encoder_config.video_format.parameters["foo"] = "foo";
1460 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001461 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001462 timestamp_ms += kFrameIntervalMs;
1463 video_source_.IncomingCapturedFrame(
1464 CreateFrame(timestamp_ms, kWidth, kHeight));
1465 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001466 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1468 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1469
mflodmancc3d4422017-08-03 08:27:51 -07001470 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001471}
1472
mflodmancc3d4422017-08-03 08:27:51 -07001473TEST_F(VideoStreamEncoderTest,
1474 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
1475 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001476
asapersson0944a802017-04-07 00:57:58 -07001477 const int kWidth = 1280;
1478 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001479 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001480
asaperssonfab67072017-04-04 05:51:49 -07001481 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001482 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001483 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001484 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001485 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001486 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1487
asapersson02465b82017-04-10 01:12:52 -07001488 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001489 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001490 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001491 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001492 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001493 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001494 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001495 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1496
1497 // Set new source with adaptation still enabled.
1498 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001499 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001500 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001501
1502 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001503 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001504 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001505 stats = stats_proxy_->GetStats();
1506 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001507 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001508 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1509
sprangc5d62e22017-04-02 23:53:04 -07001510 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001511 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001512 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001513 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001514 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001515 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001516 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001517 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001518 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001519 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001520 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1521
sprangc5d62e22017-04-02 23:53:04 -07001522 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001523 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001524 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1525 mock_stats.input_frame_rate = 30;
1526 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001527 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001528 stats_proxy_->ResetMockStats();
1529
1530 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001531 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001532 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001533
1534 // Framerate now adapted.
1535 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001536 EXPECT_FALSE(stats.cpu_limited_resolution);
1537 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001538 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1539
1540 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001541 video_stream_encoder_->SetSource(&new_video_source,
1542 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001543 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001544 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001545 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001546
1547 stats = stats_proxy_->GetStats();
1548 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001549 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001550 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1551
1552 // Try to trigger overuse. Should not succeed.
1553 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001554 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001555 stats_proxy_->ResetMockStats();
1556
1557 stats = stats_proxy_->GetStats();
1558 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001559 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001560 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1561
1562 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001563 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001564 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001565 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001566 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001567 stats = stats_proxy_->GetStats();
1568 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001569 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001570 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001571
1572 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001573 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001574 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001575 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001576 stats = stats_proxy_->GetStats();
1577 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001578 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001579 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1580
1581 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001582 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001583 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001584 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001585 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001586 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001587 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001588 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001589 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001590 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001591 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1592
1593 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001594 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001595 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001596 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001597 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001598 stats = stats_proxy_->GetStats();
1599 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001600 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001601 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001602 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001603
mflodmancc3d4422017-08-03 08:27:51 -07001604 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001605}
1606
mflodmancc3d4422017-08-03 08:27:51 -07001607TEST_F(VideoStreamEncoderTest,
1608 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001609 const int kWidth = 1280;
1610 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001611 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001612
asaperssonfab67072017-04-04 05:51:49 -07001613 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001614 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001615
asaperssonfab67072017-04-04 05:51:49 -07001616 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001617 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001618
asaperssonfab67072017-04-04 05:51:49 -07001619 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001620 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001621
asaperssonfab67072017-04-04 05:51:49 -07001622 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001623 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001624
kthelgason876222f2016-11-29 01:44:11 -08001625 // Expect a scale down.
1626 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001627 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001628
asapersson02465b82017-04-10 01:12:52 -07001629 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001630 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001631 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001632 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001633
asaperssonfab67072017-04-04 05:51:49 -07001634 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001635 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001636 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001637 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001638
asaperssonfab67072017-04-04 05:51:49 -07001639 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001640 EXPECT_EQ(std::numeric_limits<int>::max(),
1641 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001642
asaperssonfab67072017-04-04 05:51:49 -07001643 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001644 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001645 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001646 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001647
asapersson02465b82017-04-10 01:12:52 -07001648 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001649 EXPECT_EQ(std::numeric_limits<int>::max(),
1650 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001651
mflodmancc3d4422017-08-03 08:27:51 -07001652 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001653}
1654
mflodmancc3d4422017-08-03 08:27:51 -07001655TEST_F(VideoStreamEncoderTest,
1656 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001657 const int kWidth = 1280;
1658 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001659 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001660
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001661 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001662 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001663 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001664 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001665
1666 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001667 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001668 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001669 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1670 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1671
1672 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001673 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001674 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001675 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1676 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1677 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1678
1679 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001680 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001681 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1682 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1683 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1684
mflodmancc3d4422017-08-03 08:27:51 -07001685 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001686}
1687
mflodmancc3d4422017-08-03 08:27:51 -07001688TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001689 const int kWidth = 1280;
1690 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001691 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001692
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001693 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001694 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001695 video_stream_encoder_->SetSource(&source,
1696 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001697 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1698 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001699 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001700
1701 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001702 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001703 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1704 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1705 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1706 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1707
1708 // Trigger adapt down for same input resolution, expect no change.
1709 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1710 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001711 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001712 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1713 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1714 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1715
1716 // Trigger adapt down for larger input resolution, expect no change.
1717 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1718 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001719 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001720 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1721 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1722 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1723
mflodmancc3d4422017-08-03 08:27:51 -07001724 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001725}
1726
mflodmancc3d4422017-08-03 08:27:51 -07001727TEST_F(VideoStreamEncoderTest,
1728 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001729 const int kWidth = 1280;
1730 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001731 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001732
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001733 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001734 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001735 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001736 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001737
1738 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001739 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001740 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001741 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1742 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1743
1744 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001745 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001746 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001747 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1748 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1749
mflodmancc3d4422017-08-03 08:27:51 -07001750 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001751}
1752
mflodmancc3d4422017-08-03 08:27:51 -07001753TEST_F(VideoStreamEncoderTest,
1754 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001755 const int kWidth = 1280;
1756 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001757 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001758
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001759 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001760 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001761 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001762 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001763
1764 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001765 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001766 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001768 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1769
1770 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001771 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001772 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001773 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001774 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1775
mflodmancc3d4422017-08-03 08:27:51 -07001776 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001777}
1778
mflodmancc3d4422017-08-03 08:27:51 -07001779TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001780 const int kWidth = 1280;
1781 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001782 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001783
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001784 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001785 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001786 video_stream_encoder_->SetSource(&source,
1787 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001788
1789 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1790 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001791 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001792 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1793 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1794 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1795
1796 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001797 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001798 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001799 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1800 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1801 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1802
mflodmancc3d4422017-08-03 08:27:51 -07001803 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001804}
1805
mflodmancc3d4422017-08-03 08:27:51 -07001806TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001807 const int kWidth = 1280;
1808 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001809 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001810
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001811 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001812 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001813 video_stream_encoder_->SetSource(&source,
1814 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001815
1816 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1817 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001818 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001819 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1820 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1821 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1822
1823 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001824 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001825 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001826 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1827 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1828 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1829
mflodmancc3d4422017-08-03 08:27:51 -07001830 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001831}
1832
mflodmancc3d4422017-08-03 08:27:51 -07001833TEST_F(VideoStreamEncoderTest,
1834 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001835 const int kWidth = 1280;
1836 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001837 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001838
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001839 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001840 AdaptingFrameForwarder source;
1841 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001843 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001844
1845 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001846 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001847 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1849 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1850
1851 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001852 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001853 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001854 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001855 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001856 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1857 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1858
1859 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001860 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001861 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001862 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1863 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1864 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1865
mflodmancc3d4422017-08-03 08:27:51 -07001866 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001867}
1868
mflodmancc3d4422017-08-03 08:27:51 -07001869TEST_F(VideoStreamEncoderTest,
1870 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001871 const int kWidth = 1280;
1872 const int kHeight = 720;
1873 const int kInputFps = 30;
mflodmancc3d4422017-08-03 08:27:51 -07001874 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001875
1876 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1877 stats.input_frame_rate = kInputFps;
1878 stats_proxy_->SetMockStats(stats);
1879
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001880 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001881 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1882 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001883 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001884
1885 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001886 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001887 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1888 sink_.WaitForEncodedFrame(2);
1889 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1890
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001891 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001892 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001893 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001894 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001895 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001896
1897 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07001898 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001899 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1900 sink_.WaitForEncodedFrame(3);
1901 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
1902
1903 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001904 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001905 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001906
mflodmancc3d4422017-08-03 08:27:51 -07001907 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001908}
1909
mflodmancc3d4422017-08-03 08:27:51 -07001910TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07001911 const int kWidth = 1280;
1912 const int kHeight = 720;
1913 const size_t kNumFrames = 10;
1914
mflodmancc3d4422017-08-03 08:27:51 -07001915 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001916
asaperssond0de2952017-04-21 01:47:31 -07001917 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07001918 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07001919 video_source_.set_adaptation_enabled(true);
1920
1921 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1922 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1923
1924 int downscales = 0;
1925 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02001926 video_source_.IncomingCapturedFrame(
1927 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
1928 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07001929
asaperssonfab67072017-04-04 05:51:49 -07001930 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07001931 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07001932 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07001933 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07001934
1935 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
1936 ++downscales;
1937
1938 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1939 EXPECT_EQ(downscales,
1940 stats_proxy_->GetStats().number_of_quality_adapt_changes);
1941 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001942 }
mflodmancc3d4422017-08-03 08:27:51 -07001943 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07001944}
1945
mflodmancc3d4422017-08-03 08:27:51 -07001946TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07001947 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
1948 const int kWidth = 1280;
1949 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001950 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07001951
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001952 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07001953 AdaptingFrameForwarder source;
1954 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001955 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001956 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07001957
Åsa Persson8c1bf952018-09-13 10:42:19 +02001958 int64_t timestamp_ms = kFrameIntervalMs;
1959 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001960 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001961 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001962 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1963 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1964
1965 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001966 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001967 timestamp_ms += kFrameIntervalMs;
1968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1969 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001970 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001971 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1972 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1973
1974 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001975 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001976 timestamp_ms += kFrameIntervalMs;
1977 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001978 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001979 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001980 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1981 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1982
1983 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001984 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001985 timestamp_ms += kFrameIntervalMs;
1986 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1987 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001988 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001989 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1990 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1991
1992 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001993 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001994 timestamp_ms += kFrameIntervalMs;
1995 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07001996 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001997 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001998 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1999 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2000
mflodmancc3d4422017-08-03 08:27:51 -07002001 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002002}
2003
mflodmancc3d4422017-08-03 08:27:51 -07002004TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002005 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2006 const int kWidth = 1280;
2007 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002008 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002009
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002010 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002011 AdaptingFrameForwarder source;
2012 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002013 video_stream_encoder_->SetSource(&source,
2014 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002015
Åsa Persson8c1bf952018-09-13 10:42:19 +02002016 int64_t timestamp_ms = kFrameIntervalMs;
2017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002018 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002019 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002020 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2021 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2022
2023 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002024 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002025 timestamp_ms += kFrameIntervalMs;
2026 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2027 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002028 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2029 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2030 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2031
2032 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002033 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002034 timestamp_ms += kFrameIntervalMs;
2035 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002036 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002037 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002038 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2039 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2040
2041 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002042 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002043 timestamp_ms += kFrameIntervalMs;
2044 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2045 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002046 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2047 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2048 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2049
2050 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002051 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002052 timestamp_ms += kFrameIntervalMs;
2053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002054 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002055 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002056 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2057 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2058
mflodmancc3d4422017-08-03 08:27:51 -07002059 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002060}
2061
mflodmancc3d4422017-08-03 08:27:51 -07002062TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002063 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2064 const int kWidth = 1280;
2065 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002066 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002067
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002068 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002069 AdaptingFrameForwarder source;
2070 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002071 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002072 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002073
Åsa Persson8c1bf952018-09-13 10:42:19 +02002074 int64_t timestamp_ms = kFrameIntervalMs;
2075 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002076 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002077 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002078 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2079 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2080 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2081 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2082
2083 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002084 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002085 timestamp_ms += kFrameIntervalMs;
2086 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2087 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002088 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002089 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2090 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2091 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2092 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2093
2094 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002095 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002096 timestamp_ms += kFrameIntervalMs;
2097 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2098 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002099 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002100 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2101 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2102 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2103 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2104
Jonathan Yubc771b72017-12-08 17:04:29 -08002105 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002106 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002107 timestamp_ms += kFrameIntervalMs;
2108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2109 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002110 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002111 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2112 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002113 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002114 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2115
Jonathan Yubc771b72017-12-08 17:04:29 -08002116 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002117 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002118 timestamp_ms += kFrameIntervalMs;
2119 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2120 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002121 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002122 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002123 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2124 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2125 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2126 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2127
Jonathan Yubc771b72017-12-08 17:04:29 -08002128 // Trigger quality adapt down, expect no change (min resolution reached).
2129 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002130 timestamp_ms += kFrameIntervalMs;
2131 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2132 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002133 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2134 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2135 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2136 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2137 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2138
2139 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002140 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002141 timestamp_ms += kFrameIntervalMs;
2142 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2143 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002144 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002145 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2146 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2147 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2148 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2149
2150 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2151 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002152 timestamp_ms += kFrameIntervalMs;
2153 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2154 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002155 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2156 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2157 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2158 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2159 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2160
2161 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2162 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002163 timestamp_ms += kFrameIntervalMs;
2164 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2165 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002166 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002167 last_wants = source.sink_wants();
2168 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2169 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002170 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002171 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2172
2173 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002174 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002175 timestamp_ms += kFrameIntervalMs;
2176 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2177 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002178 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002179 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2180 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002181 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002182 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2183
2184 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002185 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002186 timestamp_ms += kFrameIntervalMs;
2187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002188 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002189 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002190 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002191 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2192 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002193 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002194 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002195
mflodmancc3d4422017-08-03 08:27:51 -07002196 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002197}
2198
mflodmancc3d4422017-08-03 08:27:51 -07002199TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002200 const int kWidth = 640;
2201 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002202
mflodmancc3d4422017-08-03 08:27:51 -07002203 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002204
perkj803d97f2016-11-01 11:45:46 -07002205 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002206 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002207 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002208 }
2209
mflodmancc3d4422017-08-03 08:27:51 -07002210 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002211 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002212 video_source_.IncomingCapturedFrame(CreateFrame(
2213 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002214 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002215 }
2216
mflodmancc3d4422017-08-03 08:27:51 -07002217 video_stream_encoder_->Stop();
2218 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002219 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002220
perkj803d97f2016-11-01 11:45:46 -07002221 EXPECT_EQ(1,
2222 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2223 EXPECT_EQ(
2224 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2225}
2226
mflodmancc3d4422017-08-03 08:27:51 -07002227TEST_F(VideoStreamEncoderTest,
2228 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
2229 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002230 const int kWidth = 640;
2231 const int kHeight = 360;
2232
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002233 video_stream_encoder_->SetSource(&video_source_,
2234 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002235
2236 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2237 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002238 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002239 }
2240
mflodmancc3d4422017-08-03 08:27:51 -07002241 video_stream_encoder_->Stop();
2242 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002243 stats_proxy_.reset();
2244
2245 EXPECT_EQ(0,
2246 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2247}
2248
mflodmancc3d4422017-08-03 08:27:51 -07002249TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002250 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002251 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002252
2253 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002254 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002255 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002256 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002257
sprang57c2fff2017-01-16 06:24:02 -08002258 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002259 .Times(1);
mflodmancc3d4422017-08-03 08:27:51 -07002260 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002261
sprang57c2fff2017-01-16 06:24:02 -08002262 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002263 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2264 WaitForEncodedFrame(rtc::TimeMillis());
2265 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2266 fake_encoder_.GetAndResetLastBitrateAllocation();
2267 // Check that encoder has been updated too, not just allocation observer.
2268 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
2269 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002270
2271 // Not called on second frame.
2272 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2273 .Times(0);
2274 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002275 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2276 WaitForEncodedFrame(rtc::TimeMillis());
2277 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002278
2279 // Called after a process interval.
2280 const int64_t kProcessIntervalMs =
2281 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002282 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2283 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002284 const int64_t start_time_ms = rtc::TimeMillis();
2285 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2286 video_source_.IncomingCapturedFrame(
2287 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2288 WaitForEncodedFrame(rtc::TimeMillis());
2289 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
2290 }
2291
2292 // Since rates are unchanged, encoder should not be reconfigured.
2293 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002294
mflodmancc3d4422017-08-03 08:27:51 -07002295 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002296}
2297
Niels Möller7dc26b72017-12-06 10:27:48 +01002298TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2299 const int kFrameWidth = 1280;
2300 const int kFrameHeight = 720;
2301 const int kFramerate = 24;
2302
2303 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2304 test::FrameForwarder source;
2305 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002306 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002307
2308 // Insert a single frame, triggering initial configuration.
2309 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2310 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2311
2312 EXPECT_EQ(
2313 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2314 kDefaultFramerate);
2315
2316 // Trigger reconfigure encoder (without resetting the entire instance).
2317 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002318 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002319 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2320 video_encoder_config.number_of_streams = 1;
2321 video_encoder_config.video_stream_factory =
2322 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2323 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002324 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002325 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2326
2327 // Detector should be updated with fps limit from codec config.
2328 EXPECT_EQ(
2329 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2330 kFramerate);
2331
2332 // Trigger overuse, max framerate should be reduced.
2333 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2334 stats.input_frame_rate = kFramerate;
2335 stats_proxy_->SetMockStats(stats);
2336 video_stream_encoder_->TriggerCpuOveruse();
2337 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2338 int adapted_framerate =
2339 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2340 EXPECT_LT(adapted_framerate, kFramerate);
2341
2342 // Trigger underuse, max framerate should go back to codec configured fps.
2343 // Set extra low fps, to make sure it's actually reset, not just incremented.
2344 stats = stats_proxy_->GetStats();
2345 stats.input_frame_rate = adapted_framerate / 2;
2346 stats_proxy_->SetMockStats(stats);
2347 video_stream_encoder_->TriggerCpuNormalUsage();
2348 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2349 EXPECT_EQ(
2350 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2351 kFramerate);
2352
2353 video_stream_encoder_->Stop();
2354}
2355
2356TEST_F(VideoStreamEncoderTest,
2357 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2358 const int kFrameWidth = 1280;
2359 const int kFrameHeight = 720;
2360 const int kLowFramerate = 15;
2361 const int kHighFramerate = 25;
2362
2363 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2364 test::FrameForwarder source;
2365 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002366 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002367
2368 // Trigger initial configuration.
2369 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002370 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002371 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2372 video_encoder_config.number_of_streams = 1;
2373 video_encoder_config.video_stream_factory =
2374 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2375 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2376 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002377 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002378 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2379
2380 EXPECT_EQ(
2381 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2382 kLowFramerate);
2383
2384 // Trigger overuse, max framerate should be reduced.
2385 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2386 stats.input_frame_rate = kLowFramerate;
2387 stats_proxy_->SetMockStats(stats);
2388 video_stream_encoder_->TriggerCpuOveruse();
2389 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2390 int adapted_framerate =
2391 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2392 EXPECT_LT(adapted_framerate, kLowFramerate);
2393
2394 // Reconfigure the encoder with a new (higher max framerate), max fps should
2395 // still respect the adaptation.
2396 video_encoder_config.video_stream_factory =
2397 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2398 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2399 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002400 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2402
2403 EXPECT_EQ(
2404 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2405 adapted_framerate);
2406
2407 // Trigger underuse, max framerate should go back to codec configured fps.
2408 stats = stats_proxy_->GetStats();
2409 stats.input_frame_rate = adapted_framerate;
2410 stats_proxy_->SetMockStats(stats);
2411 video_stream_encoder_->TriggerCpuNormalUsage();
2412 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2413 EXPECT_EQ(
2414 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2415 kHighFramerate);
2416
2417 video_stream_encoder_->Stop();
2418}
2419
mflodmancc3d4422017-08-03 08:27:51 -07002420TEST_F(VideoStreamEncoderTest,
2421 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002422 const int kFrameWidth = 1280;
2423 const int kFrameHeight = 720;
2424 const int kFramerate = 24;
2425
mflodmancc3d4422017-08-03 08:27:51 -07002426 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002427 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002428 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002429 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002430
2431 // Trigger initial configuration.
2432 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002433 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002434 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2435 video_encoder_config.number_of_streams = 1;
2436 video_encoder_config.video_stream_factory =
2437 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2438 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002439 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002440 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002441 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002442
Niels Möller7dc26b72017-12-06 10:27:48 +01002443 EXPECT_EQ(
2444 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2445 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002446
2447 // Trigger overuse, max framerate should be reduced.
2448 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2449 stats.input_frame_rate = kFramerate;
2450 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002451 video_stream_encoder_->TriggerCpuOveruse();
2452 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002453 int adapted_framerate =
2454 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002455 EXPECT_LT(adapted_framerate, kFramerate);
2456
2457 // Change degradation preference to not enable framerate scaling. Target
2458 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002459 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002460 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002461 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002462 EXPECT_EQ(
2463 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2464 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002465
mflodmancc3d4422017-08-03 08:27:51 -07002466 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002467}
2468
mflodmancc3d4422017-08-03 08:27:51 -07002469TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002470 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002471 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002472 const int kWidth = 640;
2473 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002474
asaperssonfab67072017-04-04 05:51:49 -07002475 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002476
2477 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002478 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002479
2480 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002481 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002482
sprangc5d62e22017-04-02 23:53:04 -07002483 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002484
asaperssonfab67072017-04-04 05:51:49 -07002485 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002486 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002487 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002488
2489 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002490 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002491
sprangc5d62e22017-04-02 23:53:04 -07002492 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002493
mflodmancc3d4422017-08-03 08:27:51 -07002494 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002495}
2496
mflodmancc3d4422017-08-03 08:27:51 -07002497TEST_F(VideoStreamEncoderTest,
2498 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002499 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002500 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002501 const int kWidth = 640;
2502 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002503
2504 // We expect the n initial frames to get dropped.
2505 int i;
2506 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002507 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002508 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002509 }
2510 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002511 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002512 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002513
2514 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002515 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002516
mflodmancc3d4422017-08-03 08:27:51 -07002517 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002518}
2519
mflodmancc3d4422017-08-03 08:27:51 -07002520TEST_F(VideoStreamEncoderTest,
2521 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002522 const int kWidth = 640;
2523 const int kHeight = 360;
mflodmancc3d4422017-08-03 08:27:51 -07002524 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002525
2526 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002528 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002529
asaperssonfab67072017-04-04 05:51:49 -07002530 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002531 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002532 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002533
mflodmancc3d4422017-08-03 08:27:51 -07002534 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002535}
2536
mflodmancc3d4422017-08-03 08:27:51 -07002537TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002538 const int kWidth = 640;
2539 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002540 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002541
2542 VideoEncoderConfig video_encoder_config;
2543 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2544 // Make format different, to force recreation of encoder.
2545 video_encoder_config.video_format.parameters["foo"] = "foo";
2546 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002547 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002548 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002549
kthelgasonb83797b2017-02-14 11:57:25 -08002550 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002551 video_stream_encoder_->SetSource(&video_source_,
2552 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002553
asaperssonfab67072017-04-04 05:51:49 -07002554 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002555 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002556 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002557
mflodmancc3d4422017-08-03 08:27:51 -07002558 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002559 fake_encoder_.SetQualityScaling(true);
2560}
2561
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002562TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2563 webrtc::test::ScopedFieldTrials field_trials(
2564 "WebRTC-InitialFramedrop/Enabled/");
2565 // Reset encoder for field trials to take effect.
2566 ConfigureEncoder(video_encoder_config_.Copy());
2567 const int kTooLowBitrateForFrameSizeBps = 10000;
2568 const int kWidth = 640;
2569 const int kHeight = 360;
2570
2571 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2572 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2573 // Frame should not be dropped.
2574 WaitForEncodedFrame(1);
2575
2576 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
2577 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2578 // Expect to drop this frame, the wait should time out.
2579 ExpectDroppedFrame();
2580
2581 // Expect the sink_wants to specify a scaled frame.
2582 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2583 video_stream_encoder_->Stop();
2584}
2585
mflodmancc3d4422017-08-03 08:27:51 -07002586TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002587 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2588 const int kTooSmallWidth = 10;
2589 const int kTooSmallHeight = 10;
mflodmancc3d4422017-08-03 08:27:51 -07002590 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002591
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002592 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002593 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002594 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002595 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002596 VerifyNoLimitation(source.sink_wants());
2597 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2598
2599 // Trigger adapt down, too small frame, expect no change.
2600 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002601 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002602 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002603 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002604 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2605 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2606
mflodmancc3d4422017-08-03 08:27:51 -07002607 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002608}
2609
mflodmancc3d4422017-08-03 08:27:51 -07002610TEST_F(VideoStreamEncoderTest,
2611 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002612 const int kTooSmallWidth = 10;
2613 const int kTooSmallHeight = 10;
2614 const int kFpsLimit = 7;
mflodmancc3d4422017-08-03 08:27:51 -07002615 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002616
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002617 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002618 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002619 video_stream_encoder_->SetSource(&source,
2620 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002621 VerifyNoLimitation(source.sink_wants());
2622 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2624
2625 // Trigger adapt down, expect limited framerate.
2626 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002627 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002628 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002629 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2630 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2632 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2633
2634 // Trigger adapt down, too small frame, expect no change.
2635 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002636 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002637 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002638 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2639 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2641 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2642
mflodmancc3d4422017-08-03 08:27:51 -07002643 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002644}
2645
mflodmancc3d4422017-08-03 08:27:51 -07002646TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002647 fake_encoder_.ForceInitEncodeFailure(true);
mflodmancc3d4422017-08-03 08:27:51 -07002648 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002649 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002650 const int kFrameWidth = 1280;
2651 const int kFrameHeight = 720;
2652 video_source_.IncomingCapturedFrame(
2653 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002654 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002655 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002656}
2657
sprangb1ca0732017-02-01 08:38:12 -08002658// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002659TEST_F(VideoStreamEncoderTest,
2660 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
2661 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002662
2663 const int kFrameWidth = 1280;
2664 const int kFrameHeight = 720;
2665 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002666 // requested by
2667 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002668 video_source_.set_adaptation_enabled(true);
2669
2670 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002671 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002672 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002673
2674 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002675 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002676 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002677 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002678 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002679
asaperssonfab67072017-04-04 05:51:49 -07002680 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002681 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002682 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002683 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002684 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002685
mflodmancc3d4422017-08-03 08:27:51 -07002686 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002687}
sprangfe627f32017-03-29 08:24:59 -07002688
mflodmancc3d4422017-08-03 08:27:51 -07002689TEST_F(VideoStreamEncoderTest,
2690 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002691 const int kFrameWidth = 1280;
2692 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002693
mflodmancc3d4422017-08-03 08:27:51 -07002694 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2695 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002696 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002697 video_source_.set_adaptation_enabled(true);
2698
sprang4847ae62017-06-27 07:06:52 -07002699 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002700
2701 video_source_.IncomingCapturedFrame(
2702 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002703 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002704
2705 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002706 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002707
2708 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002709 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002710 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002711 video_source_.IncomingCapturedFrame(
2712 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002713 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002714 }
2715
2716 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002717 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002718 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002719 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002720 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002721 video_source_.IncomingCapturedFrame(
2722 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002723 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002724 ++num_frames_dropped;
2725 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002726 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002727 }
2728 }
2729
sprang4847ae62017-06-27 07:06:52 -07002730 // Add some slack to account for frames dropped by the frame dropper.
2731 const int kErrorMargin = 1;
2732 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002733 kErrorMargin);
2734
2735 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002736 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002737 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002738 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002739 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002740 video_source_.IncomingCapturedFrame(
2741 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002742 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002743 ++num_frames_dropped;
2744 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002745 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002746 }
2747 }
sprang4847ae62017-06-27 07:06:52 -07002748 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002749 kErrorMargin);
2750
2751 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002752 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002753 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002754 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002755 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002756 video_source_.IncomingCapturedFrame(
2757 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002758 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002759 ++num_frames_dropped;
2760 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002761 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002762 }
2763 }
sprang4847ae62017-06-27 07:06:52 -07002764 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002765 kErrorMargin);
2766
2767 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002768 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002769 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002770 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002771 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002772 video_source_.IncomingCapturedFrame(
2773 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002774 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002775 ++num_frames_dropped;
2776 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002777 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002778 }
2779 }
2780 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2781
mflodmancc3d4422017-08-03 08:27:51 -07002782 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002783}
2784
mflodmancc3d4422017-08-03 08:27:51 -07002785TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002786 const int kFramerateFps = 5;
2787 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002788 const int kFrameWidth = 1280;
2789 const int kFrameHeight = 720;
2790
sprang4847ae62017-06-27 07:06:52 -07002791 // Reconfigure encoder with two temporal layers and screensharing, which will
2792 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002793 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002794
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2796 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002797 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002798 video_source_.set_adaptation_enabled(true);
2799
sprang4847ae62017-06-27 07:06:52 -07002800 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002801
2802 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002803 rtc::VideoSinkWants last_wants;
2804 do {
2805 last_wants = video_source_.sink_wants();
2806
sprangc5d62e22017-04-02 23:53:04 -07002807 // Insert frames to get a new fps estimate...
2808 for (int j = 0; j < kFramerateFps; ++j) {
2809 video_source_.IncomingCapturedFrame(
2810 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08002811 if (video_source_.last_sent_width()) {
2812 sink_.WaitForEncodedFrame(timestamp_ms);
2813 }
sprangc5d62e22017-04-02 23:53:04 -07002814 timestamp_ms += kFrameIntervalMs;
Yves Gerey665174f2018-06-19 15:03:05 +02002815 fake_clock_.AdvanceTimeMicros(kFrameIntervalMs *
2816 rtc::kNumMicrosecsPerMillisec);
sprangc5d62e22017-04-02 23:53:04 -07002817 }
2818 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08002820 } while (video_source_.sink_wants().max_framerate_fps <
2821 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002822
Jonathan Yubc771b72017-12-08 17:04:29 -08002823 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07002824
mflodmancc3d4422017-08-03 08:27:51 -07002825 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002826}
asaperssonf7e294d2017-06-13 23:25:22 -07002827
mflodmancc3d4422017-08-03 08:27:51 -07002828TEST_F(VideoStreamEncoderTest,
2829 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002830 const int kWidth = 1280;
2831 const int kHeight = 720;
2832 const int64_t kFrameIntervalMs = 150;
2833 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07002834 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002835
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002836 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002837 AdaptingFrameForwarder source;
2838 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002839 video_stream_encoder_->SetSource(&source,
2840 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002841 timestamp_ms += kFrameIntervalMs;
2842 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002843 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002844 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002845 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2846 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2847 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2848
2849 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002850 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002851 timestamp_ms += kFrameIntervalMs;
2852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002853 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002854 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2855 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2856 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2857 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2858
2859 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002860 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002861 timestamp_ms += kFrameIntervalMs;
2862 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002863 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002864 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2865 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2866 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2867 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2868
2869 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002870 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002871 timestamp_ms += kFrameIntervalMs;
2872 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002874 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2875 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2876 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2877 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2878
2879 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002880 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002881 timestamp_ms += kFrameIntervalMs;
2882 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002883 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002884 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2885 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2886 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2887 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2888
2889 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002890 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002891 timestamp_ms += kFrameIntervalMs;
2892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002893 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002894 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2896 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2897 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2898
2899 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002900 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002901 timestamp_ms += kFrameIntervalMs;
2902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002904 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2907 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2908
2909 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07002910 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002911 timestamp_ms += kFrameIntervalMs;
2912 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002913 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002914 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2915 rtc::VideoSinkWants last_wants = source.sink_wants();
2916 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2917 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2918 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2919
2920 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002921 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002922 timestamp_ms += kFrameIntervalMs;
2923 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002924 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002925 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2926 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2927 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2928 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2929
2930 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002931 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002932 timestamp_ms += kFrameIntervalMs;
2933 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002934 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002935 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2936 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2937 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2938 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2939
2940 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002941 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002942 timestamp_ms += kFrameIntervalMs;
2943 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002944 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002945 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2946 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2947 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2948 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2949
2950 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002951 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002952 timestamp_ms += kFrameIntervalMs;
2953 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002954 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002955 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2956 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2957 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2958 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2959
2960 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002961 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002962 timestamp_ms += kFrameIntervalMs;
2963 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002964 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002965 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2966 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2967 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2968 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2969
2970 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002971 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002972 timestamp_ms += kFrameIntervalMs;
2973 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002974 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002975 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
2976 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2978 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2979
2980 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002981 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002982 timestamp_ms += kFrameIntervalMs;
2983 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002984 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002985 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2986 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2987 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2988 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2989
2990 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002991 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002992 timestamp_ms += kFrameIntervalMs;
2993 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002994 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07002995 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002996 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002997 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2998 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2999 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3000
3001 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003002 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003003 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003004 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3005
mflodmancc3d4422017-08-03 08:27:51 -07003006 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003007}
3008
mflodmancc3d4422017-08-03 08:27:51 -07003009TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003010 const int kWidth = 1280;
3011 const int kHeight = 720;
3012 const int64_t kFrameIntervalMs = 150;
3013 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003014 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003015
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003016 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003017 AdaptingFrameForwarder source;
3018 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003019 video_stream_encoder_->SetSource(&source,
3020 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003021 timestamp_ms += kFrameIntervalMs;
3022 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003023 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003024 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003025 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3026 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3027 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3028 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3029 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3030 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3031
3032 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003033 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003034 timestamp_ms += kFrameIntervalMs;
3035 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003036 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003037 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3038 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3039 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3040 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3041 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3042 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3043 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3044
3045 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003046 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003047 timestamp_ms += kFrameIntervalMs;
3048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003049 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003050 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3051 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3052 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3053 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3054 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3055 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3056 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3057
3058 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003059 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003060 timestamp_ms += kFrameIntervalMs;
3061 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003062 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003063 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3064 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3065 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3066 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3067 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3068 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3069 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3070
3071 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003072 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003073 timestamp_ms += kFrameIntervalMs;
3074 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003075 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003076 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3077 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3078 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3079 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3080 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3081 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3082 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3083
3084 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003085 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003086 timestamp_ms += kFrameIntervalMs;
3087 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003088 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003089 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3090 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3091 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3092 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3093 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3094 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3095 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3096
3097 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003098 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003099 timestamp_ms += kFrameIntervalMs;
3100 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003101 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003102 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003103 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003104 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3105 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3106 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3107 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3108 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3109 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3110
3111 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003112 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003113 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003114 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3115 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3116
mflodmancc3d4422017-08-03 08:27:51 -07003117 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003118}
3119
mflodmancc3d4422017-08-03 08:27:51 -07003120TEST_F(VideoStreamEncoderTest,
3121 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003122 const int kWidth = 640;
3123 const int kHeight = 360;
3124 const int kFpsLimit = 15;
3125 const int64_t kFrameIntervalMs = 150;
3126 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003127 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003128
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003129 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003130 AdaptingFrameForwarder source;
3131 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003132 video_stream_encoder_->SetSource(&source,
3133 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003134 timestamp_ms += kFrameIntervalMs;
3135 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003136 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003137 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003138 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3139 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3140 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3141 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3142 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3143 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3144
3145 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003146 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003147 timestamp_ms += kFrameIntervalMs;
3148 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003149 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003150 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3151 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3152 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3153 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3154 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3155 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3156 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3157
3158 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003159 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003160 timestamp_ms += kFrameIntervalMs;
3161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003162 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003163 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3164 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3165 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3166 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3167 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3168 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3169 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3170
3171 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003172 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003173 timestamp_ms += kFrameIntervalMs;
3174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003175 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003176 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3177 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3179 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3180 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3181 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3182 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3183
3184 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003185 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003186 timestamp_ms += kFrameIntervalMs;
3187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003188 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003189 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003190 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3191 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3192 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3193 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3194 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3195 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3196
3197 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003198 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003199 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003200 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3201 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3202
mflodmancc3d4422017-08-03 08:27:51 -07003203 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003204}
3205
mflodmancc3d4422017-08-03 08:27:51 -07003206TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003207 // Simulates simulcast behavior and makes highest stream resolutions divisible
3208 // by 4.
3209 class CroppingVideoStreamFactory
3210 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3211 public:
3212 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3213 int framerate)
3214 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3215 EXPECT_GT(num_temporal_layers, 0u);
3216 EXPECT_GT(framerate, 0);
3217 }
3218
3219 private:
3220 std::vector<VideoStream> CreateEncoderStreams(
3221 int width,
3222 int height,
3223 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003224 std::vector<VideoStream> streams = test::CreateVideoStreams(
3225 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003226 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003227 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003228 stream.max_framerate = framerate_;
3229 }
3230 return streams;
3231 }
3232
3233 const size_t num_temporal_layers_;
3234 const int framerate_;
3235 };
3236
3237 const int kFrameWidth = 1920;
3238 const int kFrameHeight = 1080;
3239 // 3/4 of 1920.
3240 const int kAdaptedFrameWidth = 1440;
3241 // 3/4 of 1080 rounded down to multiple of 4.
3242 const int kAdaptedFrameHeight = 808;
3243 const int kFramerate = 24;
3244
mflodmancc3d4422017-08-03 08:27:51 -07003245 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003246 // Trigger reconfigure encoder (without resetting the entire instance).
3247 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003248 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003249 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3250 video_encoder_config.number_of_streams = 1;
3251 video_encoder_config.video_stream_factory =
3252 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003253 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003254 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003256
3257 video_source_.set_adaptation_enabled(true);
3258
3259 video_source_.IncomingCapturedFrame(
3260 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003261 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003262
3263 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003264 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003265 video_source_.IncomingCapturedFrame(
3266 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003267 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003268
mflodmancc3d4422017-08-03 08:27:51 -07003269 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003270}
3271
mflodmancc3d4422017-08-03 08:27:51 -07003272TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003273 const int kFrameWidth = 1280;
3274 const int kFrameHeight = 720;
3275 const int kLowFps = 2;
3276 const int kHighFps = 30;
3277
mflodmancc3d4422017-08-03 08:27:51 -07003278 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003279
3280 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3281 max_framerate_ = kLowFps;
3282
3283 // Insert 2 seconds of 2fps video.
3284 for (int i = 0; i < kLowFps * 2; ++i) {
3285 video_source_.IncomingCapturedFrame(
3286 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3287 WaitForEncodedFrame(timestamp_ms);
3288 timestamp_ms += 1000 / kLowFps;
3289 }
3290
3291 // Make sure encoder is updated with new target.
mflodmancc3d4422017-08-03 08:27:51 -07003292 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003293 video_source_.IncomingCapturedFrame(
3294 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3295 WaitForEncodedFrame(timestamp_ms);
3296 timestamp_ms += 1000 / kLowFps;
3297
3298 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3299
3300 // Insert 30fps frames for just a little more than the forced update period.
3301 const int kVcmTimerIntervalFrames =
3302 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3303 const int kFrameIntervalMs = 1000 / kHighFps;
3304 max_framerate_ = kHighFps;
3305 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3306 video_source_.IncomingCapturedFrame(
3307 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3308 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3309 // be dropped if the encoder hans't been updated with the new higher target
3310 // framerate yet, causing it to overshoot the target bitrate and then
3311 // suffering the wrath of the media optimizer.
3312 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3313 timestamp_ms += kFrameIntervalMs;
3314 }
3315
3316 // Don expect correct measurement just yet, but it should be higher than
3317 // before.
3318 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3319
mflodmancc3d4422017-08-03 08:27:51 -07003320 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003321}
3322
mflodmancc3d4422017-08-03 08:27:51 -07003323TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003324 const int kFrameWidth = 1280;
3325 const int kFrameHeight = 720;
3326 const int kTargetBitrateBps = 1000000;
3327
3328 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003329 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
mflodmancc3d4422017-08-03 08:27:51 -07003330 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3331 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003332
3333 // Insert a first video frame, causes another bitrate update.
3334 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3335 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3336 video_source_.IncomingCapturedFrame(
3337 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3338 WaitForEncodedFrame(timestamp_ms);
3339
3340 // Next, simulate video suspension due to pacer queue overrun.
mflodmancc3d4422017-08-03 08:27:51 -07003341 video_stream_encoder_->OnBitrateUpdated(0, 0, 1);
sprang4847ae62017-06-27 07:06:52 -07003342
3343 // Skip ahead until a new periodic parameter update should have occured.
3344 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
3345 fake_clock_.AdvanceTimeMicros(
3346 vcm::VCMProcessTimer::kDefaultProcessIntervalMs *
3347 rtc::kNumMicrosecsPerMillisec);
3348
3349 // Bitrate observer should not be called.
3350 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3351 video_source_.IncomingCapturedFrame(
3352 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3353 ExpectDroppedFrame();
3354
mflodmancc3d4422017-08-03 08:27:51 -07003355 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003356}
ilnik6b826ef2017-06-16 06:53:48 -07003357
Niels Möller4db138e2018-04-19 09:04:13 +02003358TEST_F(VideoStreamEncoderTest,
3359 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3360 const int kFrameWidth = 1280;
3361 const int kFrameHeight = 720;
3362 const CpuOveruseOptions default_options;
3363 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3364 video_source_.IncomingCapturedFrame(
3365 CreateFrame(1, kFrameWidth, kFrameHeight));
3366 WaitForEncodedFrame(1);
3367 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3368 .low_encode_usage_threshold_percent,
3369 default_options.low_encode_usage_threshold_percent);
3370 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3371 .high_encode_usage_threshold_percent,
3372 default_options.high_encode_usage_threshold_percent);
3373 video_stream_encoder_->Stop();
3374}
3375
3376TEST_F(VideoStreamEncoderTest,
3377 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3378 const int kFrameWidth = 1280;
3379 const int kFrameHeight = 720;
3380 CpuOveruseOptions hardware_options;
3381 hardware_options.low_encode_usage_threshold_percent = 150;
3382 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003383 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003384
3385 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3386 video_source_.IncomingCapturedFrame(
3387 CreateFrame(1, kFrameWidth, kFrameHeight));
3388 WaitForEncodedFrame(1);
3389 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3390 .low_encode_usage_threshold_percent,
3391 hardware_options.low_encode_usage_threshold_percent);
3392 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3393 .high_encode_usage_threshold_percent,
3394 hardware_options.high_encode_usage_threshold_percent);
3395 video_stream_encoder_->Stop();
3396}
3397
Niels Möller6bb5ab92019-01-11 11:11:10 +01003398TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3399 const int kFrameWidth = 320;
3400 const int kFrameHeight = 240;
3401 const int kFps = 30;
3402 const int kTargetBitrateBps = 120000;
3403 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3404
3405 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3406
3407 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3408 max_framerate_ = kFps;
3409
3410 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3411 fake_encoder_.SimulateOvershoot(1.0);
3412 int num_dropped = 0;
3413 for (int i = 0; i < kNumFramesInRun; ++i) {
3414 video_source_.IncomingCapturedFrame(
3415 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3416 // Wait up to two frame durations for a frame to arrive.
3417 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3418 ++num_dropped;
3419 }
3420 timestamp_ms += 1000 / kFps;
3421 }
3422
Erik Språnga8d48ab2019-02-08 14:17:40 +01003423 // Framerate should be measured to be near the expected target rate.
3424 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3425
3426 // Frame drops should be within 5% of expected 0%.
3427 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003428
3429 // Make encoder produce frames at double the expected bitrate during 3 seconds
3430 // of video, verify number of drops. Rate needs to be slightly changed in
3431 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003432 double overshoot_factor = 2.0;
3433 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3434 // With bitrate adjuster, when need to overshoot even more to trigger
3435 // frame dropping.
3436 overshoot_factor *= 2;
3437 }
3438 fake_encoder_.SimulateOvershoot(overshoot_factor);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003439 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps + 1000, 0, 0);
3440 num_dropped = 0;
3441 for (int i = 0; i < kNumFramesInRun; ++i) {
3442 video_source_.IncomingCapturedFrame(
3443 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3444 // Wait up to two frame durations for a frame to arrive.
3445 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3446 ++num_dropped;
3447 }
3448 timestamp_ms += 1000 / kFps;
3449 }
3450
Erik Språnga8d48ab2019-02-08 14:17:40 +01003451 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3452
3453 // Target framerate should be still be near the expected target, despite
3454 // the frame drops.
3455 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3456
3457 // Frame drops should be within 5% of expected 50%.
3458 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003459
3460 video_stream_encoder_->Stop();
3461}
3462
3463TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3464 const int kFrameWidth = 320;
3465 const int kFrameHeight = 240;
3466 const int kActualInputFps = 24;
3467 const int kTargetBitrateBps = 120000;
3468
3469 ASSERT_GT(max_framerate_, kActualInputFps);
3470
3471 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3472 max_framerate_ = kActualInputFps;
3473 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3474
3475 // Insert 3 seconds of video, with an input fps lower than configured max.
3476 for (int i = 0; i < kActualInputFps * 3; ++i) {
3477 video_source_.IncomingCapturedFrame(
3478 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3479 // Wait up to two frame durations for a frame to arrive.
3480 WaitForEncodedFrame(timestamp_ms);
3481 timestamp_ms += 1000 / kActualInputFps;
3482 }
3483
3484 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3485
3486 video_stream_encoder_->Stop();
3487}
3488
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003489TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3490 VideoFrame::UpdateRect rect;
3491 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3492
3493 fake_encoder_.BlockNextEncode();
3494 video_source_.IncomingCapturedFrame(
3495 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3496 WaitForEncodedFrame(1);
3497 // On the very first frame full update should be forced.
3498 rect = fake_encoder_.GetLastUpdateRect();
3499 EXPECT_EQ(rect.offset_x, 0);
3500 EXPECT_EQ(rect.offset_y, 0);
3501 EXPECT_EQ(rect.height, codec_height_);
3502 EXPECT_EQ(rect.width, codec_width_);
3503 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3504 // call to ContinueEncode.
3505 video_source_.IncomingCapturedFrame(
3506 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3507 ExpectDroppedFrame();
3508 video_source_.IncomingCapturedFrame(
3509 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3510 ExpectDroppedFrame();
3511 fake_encoder_.ContinueEncode();
3512 WaitForEncodedFrame(3);
3513 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3514 rect = fake_encoder_.GetLastUpdateRect();
3515 EXPECT_EQ(rect.offset_x, 1);
3516 EXPECT_EQ(rect.offset_y, 0);
3517 EXPECT_EQ(rect.width, 10);
3518 EXPECT_EQ(rect.height, 1);
3519
3520 video_source_.IncomingCapturedFrame(
3521 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3522 WaitForEncodedFrame(4);
3523 // Previous frame was encoded, so no accumulation should happen.
3524 rect = fake_encoder_.GetLastUpdateRect();
3525 EXPECT_EQ(rect.offset_x, 0);
3526 EXPECT_EQ(rect.offset_y, 0);
3527 EXPECT_EQ(rect.width, 1);
3528 EXPECT_EQ(rect.height, 1);
3529
3530 video_stream_encoder_->Stop();
3531}
3532
Erik Språngd7329ca2019-02-21 21:19:53 +01003533TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
3534 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3535
3536 // First frame is always keyframe.
3537 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3538 WaitForEncodedFrame(1);
3539 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3540 testing::ElementsAre(FrameType{kVideoFrameKey}));
3541
3542 // Insert delta frame.
3543 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3544 WaitForEncodedFrame(2);
3545 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3546 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3547
3548 // Request next frame be a key-frame.
3549 video_stream_encoder_->SendKeyFrame();
3550 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3551 WaitForEncodedFrame(3);
3552 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3553 testing::ElementsAre(FrameType{kVideoFrameKey}));
3554
3555 video_stream_encoder_->Stop();
3556}
3557
3558TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3559 // Setup simulcast with three streams.
3560 ResetEncoder("VP8", 3, 1, 1, false);
3561 video_stream_encoder_->OnBitrateUpdated(kSimulcastTargetBitrateBps, 0, 0);
3562 // Wait for all three layers before triggering event.
3563 sink_.SetNumExpectedLayers(3);
3564
3565 // First frame is always keyframe.
3566 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3567 WaitForEncodedFrame(1);
3568 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3569 testing::ElementsAreArray(
3570 {kVideoFrameKey, kVideoFrameKey, kVideoFrameKey}));
3571
3572 // Insert delta frame.
3573 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3574 WaitForEncodedFrame(2);
3575 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3576 testing::ElementsAreArray(
3577 {kVideoFrameDelta, kVideoFrameDelta, kVideoFrameDelta}));
3578
3579 // Request next frame be a key-frame.
3580 // Only first stream is configured to produce key-frame.
3581 video_stream_encoder_->SendKeyFrame();
3582 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3583 WaitForEncodedFrame(3);
3584 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3585 testing::ElementsAreArray(
3586 {kVideoFrameKey, kVideoFrameDelta, kVideoFrameDelta}));
3587
3588 video_stream_encoder_->Stop();
3589}
3590
3591TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3592 // Configure internal source factory and setup test again.
3593 encoder_factory_.SetHasInternalSource(true);
3594 ResetEncoder("VP8", 1, 1, 1, false);
3595 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3596
3597 // Call encoder directly, simulating internal source where encoded frame
3598 // callback in VideoStreamEncoder is called despite no OnFrame().
3599 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3600 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3601 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3602 testing::ElementsAre(FrameType{kVideoFrameKey}));
3603
3604 const std::vector<FrameType> kDeltaFrame = {kVideoFrameDelta};
3605 // Need to set timestamp manually since manually for injected frame.
3606 VideoFrame frame = CreateFrame(101, nullptr);
3607 frame.set_timestamp(101);
3608 fake_encoder_.InjectFrame(frame, false);
3609 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3610 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3611 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3612
3613 // Request key-frame. The forces a dummy frame down into the encoder.
3614 fake_encoder_.ExpectNullFrame();
3615 video_stream_encoder_->SendKeyFrame();
3616 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3617 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3618 testing::ElementsAre(FrameType{kVideoFrameKey}));
3619
3620 video_stream_encoder_->Stop();
3621}
Erik Språngb7cb7b52019-02-26 15:52:33 +01003622
3623TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
3624 // Configure internal source factory and setup test again.
3625 encoder_factory_.SetHasInternalSource(true);
3626 ResetEncoder("VP8", 1, 1, 1, false);
3627 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3628
3629 int64_t timestamp = 1;
3630 EncodedImage image;
3631 image.Allocate(kTargetBitrateBps / kDefaultFramerate / 8);
3632 image.capture_time_ms_ = ++timestamp;
3633 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
3634 const int64_t kEncodeFinishDelayMs = 10;
3635 image.timing_.encode_start_ms = timestamp;
3636 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
3637 fake_encoder_.InjectEncodedImage(image);
3638 // Wait for frame without incrementing clock.
3639 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3640 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
3641 // capture timestamp should be kEncodeFinishDelayMs in the past.
3642 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
3643 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
3644 kEncodeFinishDelayMs);
3645
3646 video_stream_encoder_->Stop();
3647}
perkj26091b12016-09-01 01:17:40 -07003648} // namespace webrtc