blob: 23984e9f52ea10e23765a3f050a3f57e4d95430e [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Per512ecb32016-09-23 15:52:06 +020015#include <utility>
16
Sebastian Jansson74682c12019-03-01 11:50:20 +010017#include "api/task_queue/global_task_queue_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080018#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010020#include "api/video/video_bitrate_allocation.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020021#include "api/video_codecs/create_vp8_temporal_layers.h"
22#include "api/video_codecs/vp8_temporal_layers.h"
Steve Anton10542f22019-01-11 09:11:00 -080023#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020024#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080028#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010029#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020030#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "system_wrappers/include/sleep.h"
32#include "test/encoder_settings.h"
33#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020034#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "test/frame_generator.h"
36#include "test/gmock.h"
37#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020038#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070040
41namespace webrtc {
42
sprangb1ca0732017-02-01 08:38:12 -080043using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080044using ::testing::_;
45using ::testing::Return;
kthelgason876222f2016-11-29 01:44:11 -080046
perkj803d97f2016-11-01 11:45:46 -070047namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020048const int kMinPixelsPerFrame = 320 * 180;
49const int kMinFramerateFps = 2;
50const int kMinBalancedFramerateFps = 7;
51const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080052const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010053const uint32_t kTargetBitrateBps = 1000000;
54const uint32_t kSimulcastTargetBitrateBps = 3150000;
55const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080056const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070057const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020058const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080059
perkj803d97f2016-11-01 11:45:46 -070060class TestBuffer : public webrtc::I420Buffer {
61 public:
62 TestBuffer(rtc::Event* event, int width, int height)
63 : I420Buffer(width, height), event_(event) {}
64
65 private:
66 friend class rtc::RefCountedObject<TestBuffer>;
67 ~TestBuffer() override {
68 if (event_)
69 event_->Set();
70 }
71 rtc::Event* const event_;
72};
73
Niels Möller7dc26b72017-12-06 10:27:48 +010074class CpuOveruseDetectorProxy : public OveruseFrameDetector {
75 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020076 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
77 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010078 last_target_framerate_fps_(-1) {}
79 virtual ~CpuOveruseDetectorProxy() {}
80
81 void OnTargetFramerateUpdated(int framerate_fps) override {
82 rtc::CritScope cs(&lock_);
83 last_target_framerate_fps_ = framerate_fps;
84 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
85 }
86
87 int GetLastTargetFramerate() {
88 rtc::CritScope cs(&lock_);
89 return last_target_framerate_fps_;
90 }
91
Niels Möller4db138e2018-04-19 09:04:13 +020092 CpuOveruseOptions GetOptions() { return options_; }
93
Niels Möller7dc26b72017-12-06 10:27:48 +010094 private:
95 rtc::CriticalSection lock_;
96 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
97};
98
mflodmancc3d4422017-08-03 08:27:51 -070099class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700100 public:
Niels Möller213618e2018-07-24 09:29:58 +0200101 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
102 const VideoStreamEncoderSettings& settings)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100103 : VideoStreamEncoder(Clock::GetRealTimeClock(),
104 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200105 stats_proxy,
106 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200107 std::unique_ptr<OveruseFrameDetector>(
108 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100109 new CpuOveruseDetectorProxy(stats_proxy)),
110 &GlobalTaskQueueFactory()) {}
perkj803d97f2016-11-01 11:45:46 -0700111
sprangb1ca0732017-02-01 08:38:12 -0800112 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100113 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800114 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800115 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700116 event.Set();
117 });
perkj070ba852017-02-16 15:46:27 -0800118 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700119 }
120
kthelgason2fc52542017-03-03 00:24:41 -0800121 // This is used as a synchronisation mechanism, to make sure that the
122 // encoder queue is not blocked before we start sending it frames.
123 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100124 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200125 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800126 ASSERT_TRUE(event.Wait(5000));
127 }
128
sprangb1ca0732017-02-01 08:38:12 -0800129 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800130
sprangb1ca0732017-02-01 08:38:12 -0800131 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800132
sprangb1ca0732017-02-01 08:38:12 -0800133 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800134
sprangb1ca0732017-02-01 08:38:12 -0800135 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700136
Niels Möller7dc26b72017-12-06 10:27:48 +0100137 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700138};
139
asapersson5f7226f2016-11-25 04:37:00 -0800140class VideoStreamFactory
141 : public VideoEncoderConfig::VideoStreamFactoryInterface {
142 public:
sprangfda496a2017-06-15 04:21:07 -0700143 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
144 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800145 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700146 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800147 }
148
149 private:
150 std::vector<VideoStream> CreateEncoderStreams(
151 int width,
152 int height,
153 const VideoEncoderConfig& encoder_config) override {
154 std::vector<VideoStream> streams =
155 test::CreateVideoStreams(width, height, encoder_config);
156 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100157 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700158 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800159 }
160 return streams;
161 }
sprangfda496a2017-06-15 04:21:07 -0700162
asapersson5f7226f2016-11-25 04:37:00 -0800163 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700164 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800165};
166
sprangb1ca0732017-02-01 08:38:12 -0800167class AdaptingFrameForwarder : public test::FrameForwarder {
168 public:
169 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700170 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800171
172 void set_adaptation_enabled(bool enabled) {
173 rtc::CritScope cs(&crit_);
174 adaptation_enabled_ = enabled;
175 }
176
asaperssonfab67072017-04-04 05:51:49 -0700177 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800178 rtc::CritScope cs(&crit_);
179 return adaptation_enabled_;
180 }
181
asapersson09f05612017-05-15 23:40:18 -0700182 rtc::VideoSinkWants last_wants() const {
183 rtc::CritScope cs(&crit_);
184 return last_wants_;
185 }
186
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200187 absl::optional<int> last_sent_width() const { return last_width_; }
188 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800189
sprangb1ca0732017-02-01 08:38:12 -0800190 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
191 int cropped_width = 0;
192 int cropped_height = 0;
193 int out_width = 0;
194 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700195 if (adaption_enabled()) {
196 if (adapter_.AdaptFrameResolution(
197 video_frame.width(), video_frame.height(),
198 video_frame.timestamp_us() * 1000, &cropped_width,
199 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100200 VideoFrame adapted_frame =
201 VideoFrame::Builder()
202 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
203 nullptr, out_width, out_height))
204 .set_timestamp_rtp(99)
205 .set_timestamp_ms(99)
206 .set_rotation(kVideoRotation_0)
207 .build();
sprangc5d62e22017-04-02 23:53:04 -0700208 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
209 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800210 last_width_.emplace(adapted_frame.width());
211 last_height_.emplace(adapted_frame.height());
212 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200213 last_width_ = absl::nullopt;
214 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700215 }
sprangb1ca0732017-02-01 08:38:12 -0800216 } else {
217 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800218 last_width_.emplace(video_frame.width());
219 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800220 }
221 }
222
223 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
224 const rtc::VideoSinkWants& wants) override {
225 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700226 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700227 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
228 wants.max_pixel_count,
229 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800230 test::FrameForwarder::AddOrUpdateSink(sink, wants);
231 }
sprangb1ca0732017-02-01 08:38:12 -0800232 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700233 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
234 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200235 absl::optional<int> last_width_;
236 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800237};
sprangc5d62e22017-04-02 23:53:04 -0700238
Niels Möller213618e2018-07-24 09:29:58 +0200239// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700240class MockableSendStatisticsProxy : public SendStatisticsProxy {
241 public:
242 MockableSendStatisticsProxy(Clock* clock,
243 const VideoSendStream::Config& config,
244 VideoEncoderConfig::ContentType content_type)
245 : SendStatisticsProxy(clock, config, content_type) {}
246
247 VideoSendStream::Stats GetStats() override {
248 rtc::CritScope cs(&lock_);
249 if (mock_stats_)
250 return *mock_stats_;
251 return SendStatisticsProxy::GetStats();
252 }
253
Niels Möller213618e2018-07-24 09:29:58 +0200254 int GetInputFrameRate() const override {
255 rtc::CritScope cs(&lock_);
256 if (mock_stats_)
257 return mock_stats_->input_frame_rate;
258 return SendStatisticsProxy::GetInputFrameRate();
259 }
sprangc5d62e22017-04-02 23:53:04 -0700260 void SetMockStats(const VideoSendStream::Stats& stats) {
261 rtc::CritScope cs(&lock_);
262 mock_stats_.emplace(stats);
263 }
264
265 void ResetMockStats() {
266 rtc::CritScope cs(&lock_);
267 mock_stats_.reset();
268 }
269
270 private:
271 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200272 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700273};
274
sprang4847ae62017-06-27 07:06:52 -0700275class MockBitrateObserver : public VideoBitrateAllocationObserver {
276 public:
Erik Språng566124a2018-04-23 12:32:22 +0200277 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700278};
279
perkj803d97f2016-11-01 11:45:46 -0700280} // namespace
281
mflodmancc3d4422017-08-03 08:27:51 -0700282class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700283 public:
284 static const int kDefaultTimeoutMs = 30 * 1000;
285
mflodmancc3d4422017-08-03 08:27:51 -0700286 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700287 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700288 codec_width_(320),
289 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200290 max_framerate_(kDefaultFramerate),
perkj26091b12016-09-01 01:17:40 -0700291 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200292 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800293 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700294 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700295 Clock::GetRealTimeClock(),
296 video_send_config_,
297 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700298 sink_(&fake_encoder_) {}
299
300 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700301 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700302 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200303 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800304 video_send_config_.encoder_settings.bitrate_allocator_factory =
305 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200306 video_send_config_.rtp.payload_name = "FAKE";
307 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700308
Per512ecb32016-09-23 15:52:06 +0200309 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200310 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700311 video_encoder_config.video_stream_factory =
312 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100313 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700314
315 // Framerate limit is specified by the VideoStreamFactory.
316 std::vector<VideoStream> streams =
317 video_encoder_config.video_stream_factory->CreateEncoderStreams(
318 codec_width_, codec_height_, video_encoder_config);
319 max_framerate_ = streams[0].max_framerate;
320 fake_clock_.SetTimeMicros(1234);
321
Niels Möllerf1338562018-04-26 09:51:47 +0200322 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800323 }
324
Niels Möllerf1338562018-04-26 09:51:47 +0200325 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700326 if (video_stream_encoder_)
327 video_stream_encoder_->Stop();
328 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
perkj803d97f2016-11-01 11:45:46 -0700329 stats_proxy_.get(), video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700330 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
331 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700332 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700333 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
334 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200335 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700336 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800337 }
338
339 void ResetEncoder(const std::string& payload_name,
340 size_t num_streams,
341 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700342 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700343 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200344 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800345
346 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200347 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800348 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100349 video_encoder_config.max_bitrate_bps =
350 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800351 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700352 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
353 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700354 video_encoder_config.content_type =
355 screenshare ? VideoEncoderConfig::ContentType::kScreen
356 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700357 if (payload_name == "VP9") {
358 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
359 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
360 video_encoder_config.encoder_specific_settings =
361 new rtc::RefCountedObject<
362 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
363 }
Niels Möllerf1338562018-04-26 09:51:47 +0200364 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700365 }
366
sprang57c2fff2017-01-16 06:24:02 -0800367 VideoFrame CreateFrame(int64_t ntp_time_ms,
368 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100369 VideoFrame frame =
370 VideoFrame::Builder()
371 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
372 destruction_event, codec_width_, codec_height_))
373 .set_timestamp_rtp(99)
374 .set_timestamp_ms(99)
375 .set_rotation(kVideoRotation_0)
376 .build();
sprang57c2fff2017-01-16 06:24:02 -0800377 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700378 return frame;
379 }
380
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100381 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
382 rtc::Event* destruction_event,
383 int offset_x) const {
384 VideoFrame frame =
385 VideoFrame::Builder()
386 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
387 destruction_event, codec_width_, codec_height_))
388 .set_timestamp_rtp(99)
389 .set_timestamp_ms(99)
390 .set_rotation(kVideoRotation_0)
391 .set_update_rect({offset_x, 0, 1, 1})
392 .build();
393 frame.set_ntp_time_ms(ntp_time_ms);
394 return frame;
395 }
396
sprang57c2fff2017-01-16 06:24:02 -0800397 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100398 VideoFrame frame =
399 VideoFrame::Builder()
400 .set_video_frame_buffer(
401 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
402 .set_timestamp_rtp(99)
403 .set_timestamp_ms(99)
404 .set_rotation(kVideoRotation_0)
405 .build();
sprang57c2fff2017-01-16 06:24:02 -0800406 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700407 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700408 return frame;
409 }
410
asapersson02465b82017-04-10 01:12:52 -0700411 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700412 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700413 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
414 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700415 }
416
asapersson09f05612017-05-15 23:40:18 -0700417 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
418 const rtc::VideoSinkWants& wants2) {
419 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
420 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
421 }
422
Åsa Persson8c1bf952018-09-13 10:42:19 +0200423 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
424 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
425 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
426 EXPECT_FALSE(wants.target_pixel_count);
427 }
428
asapersson09f05612017-05-15 23:40:18 -0700429 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
430 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200431 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700432 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
433 EXPECT_GT(wants1.max_pixel_count, 0);
434 }
435
436 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
437 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200438 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700439 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
440 }
441
asaperssonf7e294d2017-06-13 23:25:22 -0700442 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
443 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200444 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700445 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
446 }
447
448 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
449 const rtc::VideoSinkWants& wants2) {
450 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
451 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
452 }
453
454 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
455 const rtc::VideoSinkWants& wants2) {
456 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
457 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
458 }
459
460 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
461 const rtc::VideoSinkWants& wants2) {
462 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
463 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
464 EXPECT_GT(wants1.max_pixel_count, 0);
465 }
466
467 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
468 const rtc::VideoSinkWants& wants2) {
469 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
470 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
471 }
472
asapersson09f05612017-05-15 23:40:18 -0700473 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
474 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200475 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700476 EXPECT_LT(wants.max_pixel_count, pixel_count);
477 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700478 }
479
480 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
481 EXPECT_LT(wants.max_framerate_fps, fps);
482 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
483 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700484 }
485
asaperssonf7e294d2017-06-13 23:25:22 -0700486 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
487 int expected_fps) {
488 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
489 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
490 EXPECT_FALSE(wants.target_pixel_count);
491 }
492
Jonathan Yubc771b72017-12-08 17:04:29 -0800493 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
494 int last_frame_pixels) {
495 // Balanced mode should always scale FPS to the desired range before
496 // attempting to scale resolution.
497 int fps_limit = wants.max_framerate_fps;
498 if (last_frame_pixels <= 320 * 240) {
499 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
500 } else if (last_frame_pixels <= 480 * 270) {
501 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
502 } else if (last_frame_pixels <= 640 * 480) {
503 EXPECT_LE(15, fps_limit);
504 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200505 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800506 }
507 }
508
sprang4847ae62017-06-27 07:06:52 -0700509 void WaitForEncodedFrame(int64_t expected_ntp_time) {
510 sink_.WaitForEncodedFrame(expected_ntp_time);
511 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
512 }
513
514 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
515 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
516 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
517 return ok;
518 }
519
520 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
521 sink_.WaitForEncodedFrame(expected_width, expected_height);
522 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
523 }
524
525 void ExpectDroppedFrame() {
526 sink_.ExpectDroppedFrame();
527 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
528 }
529
530 bool WaitForFrame(int64_t timeout_ms) {
531 bool ok = sink_.WaitForFrame(timeout_ms);
532 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
533 return ok;
534 }
535
perkj26091b12016-09-01 01:17:40 -0700536 class TestEncoder : public test::FakeEncoder {
537 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100538 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700539
asaperssonfab67072017-04-04 05:51:49 -0700540 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800541 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700542 return config_;
543 }
544
545 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800546 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700547 block_next_encode_ = true;
548 }
549
Erik Språngaed30702018-11-05 12:57:17 +0100550 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800551 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100552 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100553 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100554 if (quality_scaling_) {
555 info.scaling_settings =
556 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
557 }
558 info.is_hardware_accelerated = is_hardware_accelerated_;
Erik Språngaed30702018-11-05 12:57:17 +0100559 }
560 return info;
kthelgason876222f2016-11-29 01:44:11 -0800561 }
562
Erik Språngb7cb7b52019-02-26 15:52:33 +0100563 int32_t RegisterEncodeCompleteCallback(
564 EncodedImageCallback* callback) override {
565 rtc::CritScope lock(&local_crit_sect_);
566 encoded_image_callback_ = callback;
567 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
568 }
569
perkjfa10b552016-10-02 23:45:26 -0700570 void ContinueEncode() { continue_encode_event_.Set(); }
571
572 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
573 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800574 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700575 EXPECT_EQ(timestamp_, timestamp);
576 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
577 }
578
kthelgason2fc52542017-03-03 00:24:41 -0800579 void SetQualityScaling(bool b) {
580 rtc::CritScope lock(&local_crit_sect_);
581 quality_scaling_ = b;
582 }
kthelgasonad9010c2017-02-14 00:46:51 -0800583
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100584 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
585 rtc::CritScope lock(&local_crit_sect_);
586 is_hardware_accelerated_ = is_hardware_accelerated;
587 }
588
sprangfe627f32017-03-29 08:24:59 -0700589 void ForceInitEncodeFailure(bool force_failure) {
590 rtc::CritScope lock(&local_crit_sect_);
591 force_init_encode_failed_ = force_failure;
592 }
593
Niels Möller6bb5ab92019-01-11 11:11:10 +0100594 void SimulateOvershoot(double rate_factor) {
595 rtc::CritScope lock(&local_crit_sect_);
596 rate_factor_ = rate_factor;
597 }
598
Erik Språngd7329ca2019-02-21 21:19:53 +0100599 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100600 rtc::CritScope lock(&local_crit_sect_);
601 return last_framerate_;
602 }
603
Erik Språngd7329ca2019-02-21 21:19:53 +0100604 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100605 rtc::CritScope lock(&local_crit_sect_);
606 return last_update_rect_;
607 }
608
Erik Språngd7329ca2019-02-21 21:19:53 +0100609 const std::vector<FrameType>& LastFrameTypes() const {
610 rtc::CritScope lock(&local_crit_sect_);
611 return last_frame_types_;
612 }
613
614 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
615 const std::vector<FrameType> frame_type = {keyframe ? kVideoFrameKey
616 : kVideoFrameDelta};
617 {
618 rtc::CritScope lock(&local_crit_sect_);
619 last_frame_types_ = frame_type;
620 }
621 FakeEncoder::Encode(input_image, nullptr, &frame_type);
622 }
623
Erik Språngb7cb7b52019-02-26 15:52:33 +0100624 void InjectEncodedImage(const EncodedImage& image) {
625 rtc::CritScope lock(&local_crit_sect_);
626 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
627 }
628
Erik Språngd7329ca2019-02-21 21:19:53 +0100629 void ExpectNullFrame() {
630 rtc::CritScope lock(&local_crit_sect_);
631 expect_null_frame_ = true;
632 }
633
634 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
635 auto allocation = last_bitrate_allocation_;
636 last_bitrate_allocation_.reset();
637 return allocation;
638 }
639
perkjfa10b552016-10-02 23:45:26 -0700640 private:
perkj26091b12016-09-01 01:17:40 -0700641 int32_t Encode(const VideoFrame& input_image,
642 const CodecSpecificInfo* codec_specific_info,
643 const std::vector<FrameType>* frame_types) override {
644 bool block_encode;
645 {
brandtre78d2662017-01-16 05:57:16 -0800646 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100647 if (expect_null_frame_) {
648 EXPECT_EQ(input_image.timestamp(), 0u);
649 EXPECT_EQ(input_image.width(), 1);
650 last_frame_types_ = *frame_types;
651 expect_null_frame_ = false;
652 } else {
653 EXPECT_GT(input_image.timestamp(), timestamp_);
654 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
655 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
656 }
perkj26091b12016-09-01 01:17:40 -0700657
658 timestamp_ = input_image.timestamp();
659 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700660 last_input_width_ = input_image.width();
661 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700662 block_encode = block_next_encode_;
663 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100664 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100665 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700666 }
667 int32_t result =
668 FakeEncoder::Encode(input_image, codec_specific_info, frame_types);
669 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700670 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700671 return result;
672 }
673
sprangfe627f32017-03-29 08:24:59 -0700674 int32_t InitEncode(const VideoCodec* config,
675 int32_t number_of_cores,
676 size_t max_payload_size) override {
677 int res =
678 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
679 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100680 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100681 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700682 // Simulate setting up temporal layers, in order to validate the life
683 // cycle of these objects.
684 int num_streams = std::max<int>(1, config->numberOfSimulcastStreams);
sprangfe627f32017-03-29 08:24:59 -0700685 for (int i = 0; i < num_streams; ++i) {
686 allocated_temporal_layers_.emplace_back(
Erik Språng4529fbc2018-10-12 10:30:31 +0200687 CreateVp8TemporalLayers(Vp8TemporalLayersType::kFixedPattern,
688 config->VP8().numberOfTemporalLayers));
sprangfe627f32017-03-29 08:24:59 -0700689 }
690 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100691 if (force_init_encode_failed_) {
692 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700693 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100694 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100695
Erik Språngb7cb7b52019-02-26 15:52:33 +0100696 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700697 return res;
698 }
699
Erik Språngb7cb7b52019-02-26 15:52:33 +0100700 int32_t Release() override {
701 rtc::CritScope lock(&local_crit_sect_);
702 EXPECT_NE(initialized_, EncoderState::kUninitialized);
703 initialized_ = EncoderState::kUninitialized;
704 return FakeEncoder::Release();
705 }
706
Niels Möller6bb5ab92019-01-11 11:11:10 +0100707 int32_t SetRateAllocation(const VideoBitrateAllocation& rate_allocation,
708 uint32_t framerate) {
709 rtc::CritScope lock(&local_crit_sect_);
710 VideoBitrateAllocation adjusted_rate_allocation;
711 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
712 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
713 if (rate_allocation.HasBitrate(si, ti)) {
714 adjusted_rate_allocation.SetBitrate(
715 si, ti,
716 static_cast<uint32_t>(rate_allocation.GetBitrate(si, ti) *
717 rate_factor_));
718 }
719 }
720 }
721 last_framerate_ = framerate;
Erik Språngd7329ca2019-02-21 21:19:53 +0100722 last_bitrate_allocation_ = rate_allocation;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100723 return FakeEncoder::SetRateAllocation(adjusted_rate_allocation,
724 framerate);
725 }
726
brandtre78d2662017-01-16 05:57:16 -0800727 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100728 enum class EncoderState {
729 kUninitialized,
730 kInitializationFailed,
731 kInitialized
732 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
733 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700734 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700735 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700736 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
737 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
738 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
739 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
740 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100741 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Erik Språng4529fbc2018-10-12 10:30:31 +0200742 std::vector<std::unique_ptr<Vp8TemporalLayers>> allocated_temporal_layers_
danilchapa37de392017-09-09 04:17:22 -0700743 RTC_GUARDED_BY(local_crit_sect_);
744 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100745 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
746 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100747 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100748 VideoFrame::UpdateRect last_update_rect_
749 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Erik Språngd7329ca2019-02-21 21:19:53 +0100750 std::vector<FrameType> last_frame_types_;
751 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100752 EncodedImageCallback* encoded_image_callback_
753 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
perkj26091b12016-09-01 01:17:40 -0700754 };
755
mflodmancc3d4422017-08-03 08:27:51 -0700756 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700757 public:
758 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100759 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700760
perkj26091b12016-09-01 01:17:40 -0700761 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700762 EXPECT_TRUE(
763 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
764 }
765
766 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
767 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700768 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700769 if (!encoded_frame_event_.Wait(timeout_ms))
770 return false;
perkj26091b12016-09-01 01:17:40 -0700771 {
772 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800773 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700774 }
775 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700776 return true;
perkj26091b12016-09-01 01:17:40 -0700777 }
778
sprangb1ca0732017-02-01 08:38:12 -0800779 void WaitForEncodedFrame(uint32_t expected_width,
780 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700781 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100782 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700783 }
784
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100785 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700786 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800787 uint32_t width = 0;
788 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800789 {
790 rtc::CritScope lock(&crit_);
791 width = last_width_;
792 height = last_height_;
793 }
794 EXPECT_EQ(expected_height, height);
795 EXPECT_EQ(expected_width, width);
796 }
797
kthelgason2fc52542017-03-03 00:24:41 -0800798 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800799
sprangc5d62e22017-04-02 23:53:04 -0700800 bool WaitForFrame(int64_t timeout_ms) {
801 return encoded_frame_event_.Wait(timeout_ms);
802 }
803
perkj26091b12016-09-01 01:17:40 -0700804 void SetExpectNoFrames() {
805 rtc::CritScope lock(&crit_);
806 expect_frames_ = false;
807 }
808
asaperssonfab67072017-04-04 05:51:49 -0700809 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200810 rtc::CritScope lock(&crit_);
811 return number_of_reconfigurations_;
812 }
813
asaperssonfab67072017-04-04 05:51:49 -0700814 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200815 rtc::CritScope lock(&crit_);
816 return min_transmit_bitrate_bps_;
817 }
818
Erik Språngd7329ca2019-02-21 21:19:53 +0100819 void SetNumExpectedLayers(size_t num_layers) {
820 rtc::CritScope lock(&crit_);
821 num_expected_layers_ = num_layers;
822 }
823
Erik Språngb7cb7b52019-02-26 15:52:33 +0100824 int64_t GetLastCaptureTimeMs() const {
825 rtc::CritScope lock(&crit_);
826 return last_capture_time_ms_;
827 }
828
perkj26091b12016-09-01 01:17:40 -0700829 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700830 Result OnEncodedImage(
831 const EncodedImage& encoded_image,
832 const CodecSpecificInfo* codec_specific_info,
833 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200834 rtc::CritScope lock(&crit_);
835 EXPECT_TRUE(expect_frames_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100836 uint32_t timestamp = encoded_image.Timestamp();
837 if (last_timestamp_ != timestamp) {
838 num_received_layers_ = 1;
839 } else {
840 ++num_received_layers_;
841 }
842 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100843 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -0800844 last_width_ = encoded_image._encodedWidth;
845 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +0100846 if (num_received_layers_ == num_expected_layers_) {
847 encoded_frame_event_.Set();
848 }
sprangb1ca0732017-02-01 08:38:12 -0800849 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200850 }
851
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100852 void OnEncoderConfigurationChanged(
853 std::vector<VideoStream> streams,
854 VideoEncoderConfig::ContentType content_type,
855 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200856 rtc::CriticalSection crit_;
857 ++number_of_reconfigurations_;
858 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
859 }
860
perkj26091b12016-09-01 01:17:40 -0700861 rtc::CriticalSection crit_;
862 TestEncoder* test_encoder_;
863 rtc::Event encoded_frame_event_;
sprangb1ca0732017-02-01 08:38:12 -0800864 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100865 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -0800866 uint32_t last_height_ = 0;
867 uint32_t last_width_ = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100868 size_t num_expected_layers_ = 1;
869 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700870 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200871 int number_of_reconfigurations_ = 0;
872 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700873 };
874
875 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100876 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200877 int codec_width_;
878 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700879 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -0700880 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200881 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800882 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700883 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700884 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800885 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700886 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700887 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700888};
889
mflodmancc3d4422017-08-03 08:27:51 -0700890TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
891 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100892 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700893 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700894 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700895 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700896 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700897}
898
mflodmancc3d4422017-08-03 08:27:51 -0700899TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700900 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100901 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200902 // The encoder will cache up to one frame for a short duration. Adding two
903 // frames means that the first frame will be dropped and the second frame will
904 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700905 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200906 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700907 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700908
mflodmancc3d4422017-08-03 08:27:51 -0700909 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700910
Sebastian Janssona3177052018-04-10 13:05:49 +0200911 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700912 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200913 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
914
915 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700916 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700917}
918
mflodmancc3d4422017-08-03 08:27:51 -0700919TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
920 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700921 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700922 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700923
mflodmancc3d4422017-08-03 08:27:51 -0700924 video_stream_encoder_->OnBitrateUpdated(0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +0200925 // The encoder will cache up to one frame for a short duration. Adding two
926 // frames means that the first frame will be dropped and the second frame will
927 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -0700928 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +0200929 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700930
mflodmancc3d4422017-08-03 08:27:51 -0700931 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -0700932 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +0200933 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
934 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -0700935 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700936}
937
mflodmancc3d4422017-08-03 08:27:51 -0700938TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
939 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700940 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700941 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700942
943 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -0700944 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -0700945
perkja49cbd32016-09-16 07:53:41 -0700946 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700947 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -0700948 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700949}
950
mflodmancc3d4422017-08-03 08:27:51 -0700951TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
952 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700953
perkja49cbd32016-09-16 07:53:41 -0700954 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700955 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700956
mflodmancc3d4422017-08-03 08:27:51 -0700957 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700958 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +0100959 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700960 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
961 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700962}
963
mflodmancc3d4422017-08-03 08:27:51 -0700964TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
965 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700966
967 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -0700968 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700969 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700970 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
971 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -0700972 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
973 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700974 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -0700975 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -0700976
mflodmancc3d4422017-08-03 08:27:51 -0700977 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700978}
979
mflodmancc3d4422017-08-03 08:27:51 -0700980TEST_F(VideoStreamEncoderTest,
981 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
982 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Per21d45d22016-10-30 21:37:57 +0100983 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200984
985 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200986 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700987 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +0100988 // The encoder will have been configured once when the first frame is
989 // received.
990 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200991
992 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200993 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +0200994 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -0700995 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200996 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +0200997
998 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200999 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001000 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001001 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001002 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001003
mflodmancc3d4422017-08-03 08:27:51 -07001004 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001005}
1006
mflodmancc3d4422017-08-03 08:27:51 -07001007TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
1008 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001009
1010 // Capture a frame and wait for it to synchronize with the encoder thread.
1011 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001012 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001013 // The encoder will have been configured once.
1014 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001015 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1016 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1017
1018 codec_width_ *= 2;
1019 codec_height_ *= 2;
1020 // Capture a frame with a higher resolution and wait for it to synchronize
1021 // with the encoder thread.
1022 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001023 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001024 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1025 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001026 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001027
mflodmancc3d4422017-08-03 08:27:51 -07001028 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001029}
1030
mflodmancc3d4422017-08-03 08:27:51 -07001031TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001032 EXPECT_TRUE(video_source_.has_sinks());
1033 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001034 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001035 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001036 EXPECT_FALSE(video_source_.has_sinks());
1037 EXPECT_TRUE(new_video_source.has_sinks());
1038
mflodmancc3d4422017-08-03 08:27:51 -07001039 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001040}
1041
mflodmancc3d4422017-08-03 08:27:51 -07001042TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001043 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001044 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001045 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001046 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001047}
1048
Jonathan Yubc771b72017-12-08 17:04:29 -08001049TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1050 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001051 const int kWidth = 1280;
1052 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001053
1054 // We rely on the automatic resolution adaptation, but we handle framerate
1055 // adaptation manually by mocking the stats proxy.
1056 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001057
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001058 // Enable BALANCED preference, no initial limitation.
Jonathan Yubc771b72017-12-08 17:04:29 -08001059 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001060 video_stream_encoder_->SetSource(&video_source_,
1061 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001062 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001063 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001064 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001065 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1066
Jonathan Yubc771b72017-12-08 17:04:29 -08001067 // Adapt down as far as possible.
1068 rtc::VideoSinkWants last_wants;
1069 int64_t t = 1;
1070 int loop_count = 0;
1071 do {
1072 ++loop_count;
1073 last_wants = video_source_.sink_wants();
1074
1075 // Simulate the framerate we've been asked to adapt to.
1076 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1077 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1078 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1079 mock_stats.input_frame_rate = fps;
1080 stats_proxy_->SetMockStats(mock_stats);
1081
1082 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1083 sink_.WaitForEncodedFrame(t);
1084 t += frame_interval_ms;
1085
mflodmancc3d4422017-08-03 08:27:51 -07001086 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001087 VerifyBalancedModeFpsRange(
1088 video_source_.sink_wants(),
1089 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1090 } while (video_source_.sink_wants().max_pixel_count <
1091 last_wants.max_pixel_count ||
1092 video_source_.sink_wants().max_framerate_fps <
1093 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001094
Jonathan Yubc771b72017-12-08 17:04:29 -08001095 // Verify that we've adapted all the way down.
1096 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001097 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001098 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1099 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001100 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001101 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1102 *video_source_.last_sent_height());
1103 EXPECT_EQ(kMinBalancedFramerateFps,
1104 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001105
Jonathan Yubc771b72017-12-08 17:04:29 -08001106 // Adapt back up the same number of times we adapted down.
1107 for (int i = 0; i < loop_count - 1; ++i) {
1108 last_wants = video_source_.sink_wants();
1109
1110 // Simulate the framerate we've been asked to adapt to.
1111 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1112 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1113 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1114 mock_stats.input_frame_rate = fps;
1115 stats_proxy_->SetMockStats(mock_stats);
1116
1117 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1118 sink_.WaitForEncodedFrame(t);
1119 t += frame_interval_ms;
1120
mflodmancc3d4422017-08-03 08:27:51 -07001121 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001122 VerifyBalancedModeFpsRange(
1123 video_source_.sink_wants(),
1124 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1125 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1126 last_wants.max_pixel_count ||
1127 video_source_.sink_wants().max_framerate_fps >
1128 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001129 }
1130
Åsa Persson8c1bf952018-09-13 10:42:19 +02001131 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001132 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001133 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001134 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1135 EXPECT_EQ((loop_count - 1) * 2,
1136 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001137
mflodmancc3d4422017-08-03 08:27:51 -07001138 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001139}
mflodmancc3d4422017-08-03 08:27:51 -07001140TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
1141 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001142 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001143
sprangc5d62e22017-04-02 23:53:04 -07001144 const int kFrameWidth = 1280;
1145 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001146
Åsa Persson8c1bf952018-09-13 10:42:19 +02001147 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001148
kthelgason5e13d412016-12-01 03:59:51 -08001149 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001150 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001151 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001152 frame_timestamp += kFrameIntervalMs;
1153
perkj803d97f2016-11-01 11:45:46 -07001154 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001155 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001156 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001157 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001158 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001159 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001160
asapersson0944a802017-04-07 00:57:58 -07001161 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001162 // wanted resolution.
1163 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1164 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1165 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001166 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001167
1168 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001169 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001170 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001171 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001172
sprangc5d62e22017-04-02 23:53:04 -07001173 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001174 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001175
sprangc5d62e22017-04-02 23:53:04 -07001176 // Force an input frame rate to be available, or the adaptation call won't
1177 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001178 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001179 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001180 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001181 stats_proxy_->SetMockStats(stats);
1182
mflodmancc3d4422017-08-03 08:27:51 -07001183 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001184 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001185 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001186 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001187 frame_timestamp += kFrameIntervalMs;
1188
1189 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001190 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001191 EXPECT_EQ(std::numeric_limits<int>::max(),
1192 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001193 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001194
asapersson02465b82017-04-10 01:12:52 -07001195 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001196 video_stream_encoder_->SetSource(&new_video_source,
1197 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001198 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001199
mflodmancc3d4422017-08-03 08:27:51 -07001200 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001201 new_video_source.IncomingCapturedFrame(
1202 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001203 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001204 frame_timestamp += kFrameIntervalMs;
1205
1206 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001207 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001208
1209 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001210 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001211 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001212 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1213 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001214 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001215 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001216
1217 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001218 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001219 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001220 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1221 EXPECT_EQ(std::numeric_limits<int>::max(),
1222 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001223 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001224
mflodmancc3d4422017-08-03 08:27:51 -07001225 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001226}
1227
mflodmancc3d4422017-08-03 08:27:51 -07001228TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
1229 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001230
asaperssonfab67072017-04-04 05:51:49 -07001231 const int kWidth = 1280;
1232 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001233 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001234 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001235 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1236 EXPECT_FALSE(stats.bw_limited_resolution);
1237 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1238
1239 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001240 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001241 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001242 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001243
1244 stats = stats_proxy_->GetStats();
1245 EXPECT_TRUE(stats.bw_limited_resolution);
1246 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1247
1248 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001249 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001250 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001251 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001252
1253 stats = stats_proxy_->GetStats();
1254 EXPECT_FALSE(stats.bw_limited_resolution);
1255 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1256 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1257
mflodmancc3d4422017-08-03 08:27:51 -07001258 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001259}
1260
mflodmancc3d4422017-08-03 08:27:51 -07001261TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
1262 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001263
1264 const int kWidth = 1280;
1265 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001266 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001267 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001268 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1269 EXPECT_FALSE(stats.cpu_limited_resolution);
1270 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1271
1272 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001273 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001274 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001275 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001276
1277 stats = stats_proxy_->GetStats();
1278 EXPECT_TRUE(stats.cpu_limited_resolution);
1279 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1280
1281 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001282 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001283 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001284 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001285
1286 stats = stats_proxy_->GetStats();
1287 EXPECT_FALSE(stats.cpu_limited_resolution);
1288 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001289 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001290
mflodmancc3d4422017-08-03 08:27:51 -07001291 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001292}
1293
mflodmancc3d4422017-08-03 08:27:51 -07001294TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
1295 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001296
asaperssonfab67072017-04-04 05:51:49 -07001297 const int kWidth = 1280;
1298 const int kHeight = 720;
1299 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001300 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001301 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001302 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001303 EXPECT_FALSE(stats.cpu_limited_resolution);
1304 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1305
asaperssonfab67072017-04-04 05:51:49 -07001306 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001307 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001308 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001309 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001310 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001311 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001312 EXPECT_TRUE(stats.cpu_limited_resolution);
1313 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1314
1315 // Set new source with adaptation still enabled.
1316 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001317 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001318 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001319
asaperssonfab67072017-04-04 05:51:49 -07001320 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001321 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001322 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001323 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001324 EXPECT_TRUE(stats.cpu_limited_resolution);
1325 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1326
1327 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001328 video_stream_encoder_->SetSource(&new_video_source,
1329 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001330
asaperssonfab67072017-04-04 05:51:49 -07001331 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001332 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001333 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001334 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001335 EXPECT_FALSE(stats.cpu_limited_resolution);
1336 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1337
1338 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001339 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001340 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001341
asaperssonfab67072017-04-04 05:51:49 -07001342 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001343 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001344 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001345 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001346 EXPECT_TRUE(stats.cpu_limited_resolution);
1347 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1348
asaperssonfab67072017-04-04 05:51:49 -07001349 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001350 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001351 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001352 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001353 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001354 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001355 EXPECT_FALSE(stats.cpu_limited_resolution);
1356 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001357 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001358
mflodmancc3d4422017-08-03 08:27:51 -07001359 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001360}
1361
mflodmancc3d4422017-08-03 08:27:51 -07001362TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
1363 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001364
asaperssonfab67072017-04-04 05:51:49 -07001365 const int kWidth = 1280;
1366 const int kHeight = 720;
1367 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001368 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001369 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001370 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001371 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001372 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001373
1374 // Set new source with adaptation still enabled.
1375 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001376 video_stream_encoder_->SetSource(&new_video_source,
1377 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001378
asaperssonfab67072017-04-04 05:51:49 -07001379 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001380 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001381 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001382 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001383 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001384 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001385
asaperssonfab67072017-04-04 05:51:49 -07001386 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001387 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001388 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001389 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001390 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001391 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001392 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001393 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001394
asaperssonfab67072017-04-04 05:51:49 -07001395 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001396 video_stream_encoder_->SetSource(&new_video_source,
1397 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001398
asaperssonfab67072017-04-04 05:51:49 -07001399 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001400 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001401 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001402 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001403 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001404 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001405
asapersson02465b82017-04-10 01:12:52 -07001406 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001407 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001408 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001409
asaperssonfab67072017-04-04 05:51:49 -07001410 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001411 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001412 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001413 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001414 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001415 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1416 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001417
mflodmancc3d4422017-08-03 08:27:51 -07001418 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001419}
1420
mflodmancc3d4422017-08-03 08:27:51 -07001421TEST_F(VideoStreamEncoderTest,
1422 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
1423 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001424
1425 const int kWidth = 1280;
1426 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001427 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001428 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001429 video_source_.IncomingCapturedFrame(
1430 CreateFrame(timestamp_ms, kWidth, kHeight));
1431 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001432 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1434 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1435
1436 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001437 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001438 timestamp_ms += kFrameIntervalMs;
1439 video_source_.IncomingCapturedFrame(
1440 CreateFrame(timestamp_ms, kWidth, kHeight));
1441 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1443 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1445
1446 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001447 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001448 timestamp_ms += kFrameIntervalMs;
1449 video_source_.IncomingCapturedFrame(
1450 CreateFrame(timestamp_ms, kWidth, kHeight));
1451 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001452 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1454 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1455
Niels Möller4db138e2018-04-19 09:04:13 +02001456 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001457 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001458
1459 VideoEncoderConfig video_encoder_config;
1460 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1461 // Make format different, to force recreation of encoder.
1462 video_encoder_config.video_format.parameters["foo"] = "foo";
1463 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001464 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001465 timestamp_ms += kFrameIntervalMs;
1466 video_source_.IncomingCapturedFrame(
1467 CreateFrame(timestamp_ms, kWidth, kHeight));
1468 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001469 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1470 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1471 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1472
mflodmancc3d4422017-08-03 08:27:51 -07001473 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001474}
1475
mflodmancc3d4422017-08-03 08:27:51 -07001476TEST_F(VideoStreamEncoderTest,
1477 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
1478 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001479
asapersson0944a802017-04-07 00:57:58 -07001480 const int kWidth = 1280;
1481 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001482 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001483
asaperssonfab67072017-04-04 05:51:49 -07001484 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001485 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001486 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001487 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001488 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001489 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1490
asapersson02465b82017-04-10 01:12:52 -07001491 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001492 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001493 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001494 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001495 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001496 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001497 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001498 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1499
1500 // Set new source with adaptation still enabled.
1501 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001502 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001503 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001504
1505 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001506 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001507 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001508 stats = stats_proxy_->GetStats();
1509 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001510 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001511 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1512
sprangc5d62e22017-04-02 23:53:04 -07001513 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001514 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001515 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001516 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001517 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001518 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001519 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001520 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001521 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001522 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001523 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1524
sprangc5d62e22017-04-02 23:53:04 -07001525 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001526 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001527 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1528 mock_stats.input_frame_rate = 30;
1529 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001530 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001531 stats_proxy_->ResetMockStats();
1532
1533 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001534 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001535 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001536
1537 // Framerate now adapted.
1538 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001539 EXPECT_FALSE(stats.cpu_limited_resolution);
1540 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001541 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1542
1543 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001544 video_stream_encoder_->SetSource(&new_video_source,
1545 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001546 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001547 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001548 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001549
1550 stats = stats_proxy_->GetStats();
1551 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001552 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001553 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1554
1555 // Try to trigger overuse. Should not succeed.
1556 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001557 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001558 stats_proxy_->ResetMockStats();
1559
1560 stats = stats_proxy_->GetStats();
1561 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001562 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001563 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1564
1565 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001566 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001567 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001568 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001569 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001570 stats = stats_proxy_->GetStats();
1571 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001572 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001573 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001574
1575 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001576 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001577 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001578 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001579 stats = stats_proxy_->GetStats();
1580 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001581 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001582 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1583
1584 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001585 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001586 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001587 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001588 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001589 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001590 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001591 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001592 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001593 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001594 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1595
1596 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001597 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001598 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001599 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001600 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001601 stats = stats_proxy_->GetStats();
1602 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001603 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001604 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001605 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001606
mflodmancc3d4422017-08-03 08:27:51 -07001607 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001608}
1609
mflodmancc3d4422017-08-03 08:27:51 -07001610TEST_F(VideoStreamEncoderTest,
1611 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001612 const int kWidth = 1280;
1613 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001614 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001615
asaperssonfab67072017-04-04 05:51:49 -07001616 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001617 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001618
asaperssonfab67072017-04-04 05:51:49 -07001619 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001620 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001621
asaperssonfab67072017-04-04 05:51:49 -07001622 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001623 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001624
asaperssonfab67072017-04-04 05:51:49 -07001625 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001626 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001627
kthelgason876222f2016-11-29 01:44:11 -08001628 // Expect a scale down.
1629 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001630 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001631
asapersson02465b82017-04-10 01:12:52 -07001632 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001633 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001634 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001635 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001636
asaperssonfab67072017-04-04 05:51:49 -07001637 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001638 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001639 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001640 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001641
asaperssonfab67072017-04-04 05:51:49 -07001642 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001643 EXPECT_EQ(std::numeric_limits<int>::max(),
1644 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001645
asaperssonfab67072017-04-04 05:51:49 -07001646 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001647 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001648 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001649 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001650
asapersson02465b82017-04-10 01:12:52 -07001651 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001652 EXPECT_EQ(std::numeric_limits<int>::max(),
1653 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001654
mflodmancc3d4422017-08-03 08:27:51 -07001655 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001656}
1657
mflodmancc3d4422017-08-03 08:27:51 -07001658TEST_F(VideoStreamEncoderTest,
1659 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001660 const int kWidth = 1280;
1661 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001662 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001663
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001664 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001665 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001666 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001667 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001668
1669 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001670 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001671 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001672 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1673 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1674
1675 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001676 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001677 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001678 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1679 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1680 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1681
1682 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001683 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001684 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1685 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1686 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1687
mflodmancc3d4422017-08-03 08:27:51 -07001688 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001689}
1690
mflodmancc3d4422017-08-03 08:27:51 -07001691TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001692 const int kWidth = 1280;
1693 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001694 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001695
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001696 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001697 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001698 video_stream_encoder_->SetSource(&source,
1699 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001700 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1701 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001702 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001703
1704 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001705 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001706 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1707 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1708 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1709 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1710
1711 // Trigger adapt down for same input resolution, expect no change.
1712 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1713 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001714 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001715 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1716 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1717 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1718
1719 // Trigger adapt down for larger input resolution, expect no change.
1720 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1721 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001722 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001723 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1724 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1725 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1726
mflodmancc3d4422017-08-03 08:27:51 -07001727 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001728}
1729
mflodmancc3d4422017-08-03 08:27:51 -07001730TEST_F(VideoStreamEncoderTest,
1731 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001732 const int kWidth = 1280;
1733 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001734 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001735
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001736 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001737 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001738 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001739 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001740
1741 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001742 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001743 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1745 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1746
1747 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001748 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001749 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001750 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1751 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1752
mflodmancc3d4422017-08-03 08:27:51 -07001753 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001754}
1755
mflodmancc3d4422017-08-03 08:27:51 -07001756TEST_F(VideoStreamEncoderTest,
1757 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001758 const int kWidth = 1280;
1759 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001760 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001761
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001762 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001763 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001764 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001765 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001766
1767 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001768 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001769 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001770 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001771 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1772
1773 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001774 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001775 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001776 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001777 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1778
mflodmancc3d4422017-08-03 08:27:51 -07001779 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001780}
1781
mflodmancc3d4422017-08-03 08:27:51 -07001782TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001783 const int kWidth = 1280;
1784 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001785 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001786
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001787 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001788 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001789 video_stream_encoder_->SetSource(&source,
1790 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001791
1792 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1793 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001794 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001795 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1796 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1797 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1798
1799 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001800 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001801 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001802 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1803 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1804 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1805
mflodmancc3d4422017-08-03 08:27:51 -07001806 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001807}
1808
mflodmancc3d4422017-08-03 08:27:51 -07001809TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001810 const int kWidth = 1280;
1811 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001812 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001813
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001814 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001815 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001816 video_stream_encoder_->SetSource(&source,
1817 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001818
1819 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1820 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001821 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001822 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1823 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1824 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1825
1826 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001827 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001828 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1830 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1831 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1832
mflodmancc3d4422017-08-03 08:27:51 -07001833 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001834}
1835
mflodmancc3d4422017-08-03 08:27:51 -07001836TEST_F(VideoStreamEncoderTest,
1837 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001838 const int kWidth = 1280;
1839 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001840 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001841
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001842 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001843 AdaptingFrameForwarder source;
1844 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001845 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001846 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001847
1848 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001849 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001850 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1852 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1853
1854 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001855 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001856 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001857 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001858 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001859 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1861
1862 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001863 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001864 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1866 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1867 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1868
mflodmancc3d4422017-08-03 08:27:51 -07001869 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001870}
1871
mflodmancc3d4422017-08-03 08:27:51 -07001872TEST_F(VideoStreamEncoderTest,
1873 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001874 const int kWidth = 1280;
1875 const int kHeight = 720;
1876 const int kInputFps = 30;
mflodmancc3d4422017-08-03 08:27:51 -07001877 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001878
1879 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1880 stats.input_frame_rate = kInputFps;
1881 stats_proxy_->SetMockStats(stats);
1882
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001883 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001884 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1885 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001886 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001887
1888 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001889 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001890 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1891 sink_.WaitForEncodedFrame(2);
1892 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1893
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001894 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001895 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001896 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001897 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001898 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001899
1900 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07001901 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001902 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1903 sink_.WaitForEncodedFrame(3);
1904 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
1905
1906 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001907 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001908 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001909
mflodmancc3d4422017-08-03 08:27:51 -07001910 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001911}
1912
mflodmancc3d4422017-08-03 08:27:51 -07001913TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07001914 const int kWidth = 1280;
1915 const int kHeight = 720;
1916 const size_t kNumFrames = 10;
1917
mflodmancc3d4422017-08-03 08:27:51 -07001918 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001919
asaperssond0de2952017-04-21 01:47:31 -07001920 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07001921 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07001922 video_source_.set_adaptation_enabled(true);
1923
1924 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1925 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1926
1927 int downscales = 0;
1928 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02001929 video_source_.IncomingCapturedFrame(
1930 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
1931 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07001932
asaperssonfab67072017-04-04 05:51:49 -07001933 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07001934 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07001935 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07001936 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07001937
1938 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
1939 ++downscales;
1940
1941 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1942 EXPECT_EQ(downscales,
1943 stats_proxy_->GetStats().number_of_quality_adapt_changes);
1944 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001945 }
mflodmancc3d4422017-08-03 08:27:51 -07001946 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07001947}
1948
mflodmancc3d4422017-08-03 08:27:51 -07001949TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07001950 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
1951 const int kWidth = 1280;
1952 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001953 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07001954
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001955 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07001956 AdaptingFrameForwarder source;
1957 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001958 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001959 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07001960
Åsa Persson8c1bf952018-09-13 10:42:19 +02001961 int64_t timestamp_ms = kFrameIntervalMs;
1962 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001963 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001964 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001965 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1966 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1967
1968 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001969 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001970 timestamp_ms += kFrameIntervalMs;
1971 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1972 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001973 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001974 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1975 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1976
1977 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001978 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001979 timestamp_ms += kFrameIntervalMs;
1980 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001981 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001982 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001983 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1984 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1985
1986 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001987 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001988 timestamp_ms += kFrameIntervalMs;
1989 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1990 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001991 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001992 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1993 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1994
1995 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001996 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001997 timestamp_ms += kFrameIntervalMs;
1998 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07001999 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002000 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002001 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2002 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2003
mflodmancc3d4422017-08-03 08:27:51 -07002004 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002005}
2006
mflodmancc3d4422017-08-03 08:27:51 -07002007TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002008 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2009 const int kWidth = 1280;
2010 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002011 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002012
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002013 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002014 AdaptingFrameForwarder source;
2015 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002016 video_stream_encoder_->SetSource(&source,
2017 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002018
Åsa Persson8c1bf952018-09-13 10:42:19 +02002019 int64_t timestamp_ms = kFrameIntervalMs;
2020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002021 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002022 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002023 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2024 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2025
2026 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002027 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002028 timestamp_ms += kFrameIntervalMs;
2029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2030 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002031 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2032 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2033 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2034
2035 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002036 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002037 timestamp_ms += kFrameIntervalMs;
2038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002039 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002040 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2042 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2043
2044 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002045 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002046 timestamp_ms += kFrameIntervalMs;
2047 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2048 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002049 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2050 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2051 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2052
2053 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002054 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002055 timestamp_ms += kFrameIntervalMs;
2056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002057 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002058 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002059 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2060 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2061
mflodmancc3d4422017-08-03 08:27:51 -07002062 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002063}
2064
mflodmancc3d4422017-08-03 08:27:51 -07002065TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002066 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2067 const int kWidth = 1280;
2068 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002069 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002070
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002071 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002072 AdaptingFrameForwarder source;
2073 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002074 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002075 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002076
Åsa Persson8c1bf952018-09-13 10:42:19 +02002077 int64_t timestamp_ms = kFrameIntervalMs;
2078 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002079 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002080 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002081 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2082 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2083 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2084 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2085
2086 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002087 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002088 timestamp_ms += kFrameIntervalMs;
2089 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2090 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002091 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002092 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2093 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2094 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2095 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2096
2097 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002098 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002099 timestamp_ms += kFrameIntervalMs;
2100 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2101 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002102 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002103 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2104 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2105 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2106 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2107
Jonathan Yubc771b72017-12-08 17:04:29 -08002108 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002109 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002110 timestamp_ms += kFrameIntervalMs;
2111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2112 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002113 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002114 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2115 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002116 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002117 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2118
Jonathan Yubc771b72017-12-08 17:04:29 -08002119 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002120 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002121 timestamp_ms += kFrameIntervalMs;
2122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2123 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002124 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002125 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002126 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2127 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2128 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2129 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2130
Jonathan Yubc771b72017-12-08 17:04:29 -08002131 // Trigger quality adapt down, expect no change (min resolution reached).
2132 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002133 timestamp_ms += kFrameIntervalMs;
2134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2135 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002136 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2137 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2138 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2139 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2140 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2141
2142 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002143 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002144 timestamp_ms += kFrameIntervalMs;
2145 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2146 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002147 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002148 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2149 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2150 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2151 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2152
2153 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2154 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002155 timestamp_ms += kFrameIntervalMs;
2156 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2157 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002158 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2159 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2160 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2161 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2162 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2163
2164 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2165 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002166 timestamp_ms += kFrameIntervalMs;
2167 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2168 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002169 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002170 last_wants = source.sink_wants();
2171 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2172 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002173 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002174 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2175
2176 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002177 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002178 timestamp_ms += kFrameIntervalMs;
2179 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2180 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002181 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002182 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2183 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002184 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002185 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2186
2187 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002188 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002189 timestamp_ms += kFrameIntervalMs;
2190 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002191 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002192 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002193 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002194 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2195 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002196 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002197 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002198
mflodmancc3d4422017-08-03 08:27:51 -07002199 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002200}
2201
mflodmancc3d4422017-08-03 08:27:51 -07002202TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002203 const int kWidth = 640;
2204 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002205
mflodmancc3d4422017-08-03 08:27:51 -07002206 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002207
perkj803d97f2016-11-01 11:45:46 -07002208 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002209 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002210 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002211 }
2212
mflodmancc3d4422017-08-03 08:27:51 -07002213 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002214 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002215 video_source_.IncomingCapturedFrame(CreateFrame(
2216 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002217 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002218 }
2219
mflodmancc3d4422017-08-03 08:27:51 -07002220 video_stream_encoder_->Stop();
2221 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002222 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002223
perkj803d97f2016-11-01 11:45:46 -07002224 EXPECT_EQ(1,
2225 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2226 EXPECT_EQ(
2227 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2228}
2229
mflodmancc3d4422017-08-03 08:27:51 -07002230TEST_F(VideoStreamEncoderTest,
2231 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
2232 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002233 const int kWidth = 640;
2234 const int kHeight = 360;
2235
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002236 video_stream_encoder_->SetSource(&video_source_,
2237 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002238
2239 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2240 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002241 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002242 }
2243
mflodmancc3d4422017-08-03 08:27:51 -07002244 video_stream_encoder_->Stop();
2245 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002246 stats_proxy_.reset();
2247
2248 EXPECT_EQ(0,
2249 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2250}
2251
mflodmancc3d4422017-08-03 08:27:51 -07002252TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002253 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002254 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002255
2256 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002257 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002258 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002259 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002260
sprang57c2fff2017-01-16 06:24:02 -08002261 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002262 .Times(1);
mflodmancc3d4422017-08-03 08:27:51 -07002263 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002264
sprang57c2fff2017-01-16 06:24:02 -08002265 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002266 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2267 WaitForEncodedFrame(rtc::TimeMillis());
2268 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2269 fake_encoder_.GetAndResetLastBitrateAllocation();
2270 // Check that encoder has been updated too, not just allocation observer.
2271 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
2272 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002273
2274 // Not called on second frame.
2275 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2276 .Times(0);
2277 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002278 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2279 WaitForEncodedFrame(rtc::TimeMillis());
2280 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002281
2282 // Called after a process interval.
2283 const int64_t kProcessIntervalMs =
2284 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002285 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2286 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002287 const int64_t start_time_ms = rtc::TimeMillis();
2288 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2289 video_source_.IncomingCapturedFrame(
2290 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2291 WaitForEncodedFrame(rtc::TimeMillis());
2292 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
2293 }
2294
2295 // Since rates are unchanged, encoder should not be reconfigured.
2296 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002297
mflodmancc3d4422017-08-03 08:27:51 -07002298 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002299}
2300
Niels Möller7dc26b72017-12-06 10:27:48 +01002301TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2302 const int kFrameWidth = 1280;
2303 const int kFrameHeight = 720;
2304 const int kFramerate = 24;
2305
2306 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2307 test::FrameForwarder source;
2308 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002309 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002310
2311 // Insert a single frame, triggering initial configuration.
2312 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2314
2315 EXPECT_EQ(
2316 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2317 kDefaultFramerate);
2318
2319 // Trigger reconfigure encoder (without resetting the entire instance).
2320 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002321 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002322 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2323 video_encoder_config.number_of_streams = 1;
2324 video_encoder_config.video_stream_factory =
2325 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2326 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002327 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2329
2330 // Detector should be updated with fps limit from codec config.
2331 EXPECT_EQ(
2332 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2333 kFramerate);
2334
2335 // Trigger overuse, max framerate should be reduced.
2336 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2337 stats.input_frame_rate = kFramerate;
2338 stats_proxy_->SetMockStats(stats);
2339 video_stream_encoder_->TriggerCpuOveruse();
2340 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2341 int adapted_framerate =
2342 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2343 EXPECT_LT(adapted_framerate, kFramerate);
2344
2345 // Trigger underuse, max framerate should go back to codec configured fps.
2346 // Set extra low fps, to make sure it's actually reset, not just incremented.
2347 stats = stats_proxy_->GetStats();
2348 stats.input_frame_rate = adapted_framerate / 2;
2349 stats_proxy_->SetMockStats(stats);
2350 video_stream_encoder_->TriggerCpuNormalUsage();
2351 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2352 EXPECT_EQ(
2353 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2354 kFramerate);
2355
2356 video_stream_encoder_->Stop();
2357}
2358
2359TEST_F(VideoStreamEncoderTest,
2360 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2361 const int kFrameWidth = 1280;
2362 const int kFrameHeight = 720;
2363 const int kLowFramerate = 15;
2364 const int kHighFramerate = 25;
2365
2366 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2367 test::FrameForwarder source;
2368 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002369 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002370
2371 // Trigger initial configuration.
2372 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002373 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002374 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2375 video_encoder_config.number_of_streams = 1;
2376 video_encoder_config.video_stream_factory =
2377 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2378 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2379 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002380 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002381 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2382
2383 EXPECT_EQ(
2384 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2385 kLowFramerate);
2386
2387 // Trigger overuse, max framerate should be reduced.
2388 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2389 stats.input_frame_rate = kLowFramerate;
2390 stats_proxy_->SetMockStats(stats);
2391 video_stream_encoder_->TriggerCpuOveruse();
2392 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2393 int adapted_framerate =
2394 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2395 EXPECT_LT(adapted_framerate, kLowFramerate);
2396
2397 // Reconfigure the encoder with a new (higher max framerate), max fps should
2398 // still respect the adaptation.
2399 video_encoder_config.video_stream_factory =
2400 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2401 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2402 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002403 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002404 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2405
2406 EXPECT_EQ(
2407 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2408 adapted_framerate);
2409
2410 // Trigger underuse, max framerate should go back to codec configured fps.
2411 stats = stats_proxy_->GetStats();
2412 stats.input_frame_rate = adapted_framerate;
2413 stats_proxy_->SetMockStats(stats);
2414 video_stream_encoder_->TriggerCpuNormalUsage();
2415 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2416 EXPECT_EQ(
2417 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2418 kHighFramerate);
2419
2420 video_stream_encoder_->Stop();
2421}
2422
mflodmancc3d4422017-08-03 08:27:51 -07002423TEST_F(VideoStreamEncoderTest,
2424 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002425 const int kFrameWidth = 1280;
2426 const int kFrameHeight = 720;
2427 const int kFramerate = 24;
2428
mflodmancc3d4422017-08-03 08:27:51 -07002429 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002430 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002431 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002432 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002433
2434 // Trigger initial configuration.
2435 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002436 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002437 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2438 video_encoder_config.number_of_streams = 1;
2439 video_encoder_config.video_stream_factory =
2440 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2441 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002442 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002443 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002444 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002445
Niels Möller7dc26b72017-12-06 10:27:48 +01002446 EXPECT_EQ(
2447 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2448 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002449
2450 // Trigger overuse, max framerate should be reduced.
2451 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2452 stats.input_frame_rate = kFramerate;
2453 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002454 video_stream_encoder_->TriggerCpuOveruse();
2455 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002456 int adapted_framerate =
2457 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002458 EXPECT_LT(adapted_framerate, kFramerate);
2459
2460 // Change degradation preference to not enable framerate scaling. Target
2461 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002462 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002463 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002464 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002465 EXPECT_EQ(
2466 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2467 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002468
mflodmancc3d4422017-08-03 08:27:51 -07002469 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002470}
2471
mflodmancc3d4422017-08-03 08:27:51 -07002472TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002473 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002474 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002475 const int kWidth = 640;
2476 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002477
asaperssonfab67072017-04-04 05:51:49 -07002478 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002479
2480 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002481 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002482
2483 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002484 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002485
sprangc5d62e22017-04-02 23:53:04 -07002486 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002487
asaperssonfab67072017-04-04 05:51:49 -07002488 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002489 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002490 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002491
2492 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002493 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002494
sprangc5d62e22017-04-02 23:53:04 -07002495 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002496
mflodmancc3d4422017-08-03 08:27:51 -07002497 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002498}
2499
mflodmancc3d4422017-08-03 08:27:51 -07002500TEST_F(VideoStreamEncoderTest,
2501 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002502 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002503 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002504 const int kWidth = 640;
2505 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002506
2507 // We expect the n initial frames to get dropped.
2508 int i;
2509 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002510 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002511 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002512 }
2513 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002514 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002515 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002516
2517 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002518 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002519
mflodmancc3d4422017-08-03 08:27:51 -07002520 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002521}
2522
mflodmancc3d4422017-08-03 08:27:51 -07002523TEST_F(VideoStreamEncoderTest,
2524 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002525 const int kWidth = 640;
2526 const int kHeight = 360;
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002528
2529 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002530 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002531 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002532
asaperssonfab67072017-04-04 05:51:49 -07002533 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002534 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002535 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002536
mflodmancc3d4422017-08-03 08:27:51 -07002537 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002538}
2539
mflodmancc3d4422017-08-03 08:27:51 -07002540TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002541 const int kWidth = 640;
2542 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002543 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002544
2545 VideoEncoderConfig video_encoder_config;
2546 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2547 // Make format different, to force recreation of encoder.
2548 video_encoder_config.video_format.parameters["foo"] = "foo";
2549 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002550 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002551 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002552
kthelgasonb83797b2017-02-14 11:57:25 -08002553 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002554 video_stream_encoder_->SetSource(&video_source_,
2555 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002556
asaperssonfab67072017-04-04 05:51:49 -07002557 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002558 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002559 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002560
mflodmancc3d4422017-08-03 08:27:51 -07002561 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002562 fake_encoder_.SetQualityScaling(true);
2563}
2564
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002565TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2566 webrtc::test::ScopedFieldTrials field_trials(
2567 "WebRTC-InitialFramedrop/Enabled/");
2568 // Reset encoder for field trials to take effect.
2569 ConfigureEncoder(video_encoder_config_.Copy());
2570 const int kTooLowBitrateForFrameSizeBps = 10000;
2571 const int kWidth = 640;
2572 const int kHeight = 360;
2573
2574 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2575 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2576 // Frame should not be dropped.
2577 WaitForEncodedFrame(1);
2578
2579 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
2580 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2581 // Expect to drop this frame, the wait should time out.
2582 ExpectDroppedFrame();
2583
2584 // Expect the sink_wants to specify a scaled frame.
2585 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2586 video_stream_encoder_->Stop();
2587}
2588
mflodmancc3d4422017-08-03 08:27:51 -07002589TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002590 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2591 const int kTooSmallWidth = 10;
2592 const int kTooSmallHeight = 10;
mflodmancc3d4422017-08-03 08:27:51 -07002593 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002594
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002595 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002596 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002597 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002598 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002599 VerifyNoLimitation(source.sink_wants());
2600 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2601
2602 // Trigger adapt down, too small frame, expect no change.
2603 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002604 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002605 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002606 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2608 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2609
mflodmancc3d4422017-08-03 08:27:51 -07002610 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002611}
2612
mflodmancc3d4422017-08-03 08:27:51 -07002613TEST_F(VideoStreamEncoderTest,
2614 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002615 const int kTooSmallWidth = 10;
2616 const int kTooSmallHeight = 10;
2617 const int kFpsLimit = 7;
mflodmancc3d4422017-08-03 08:27:51 -07002618 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002619
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002620 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002621 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002622 video_stream_encoder_->SetSource(&source,
2623 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002624 VerifyNoLimitation(source.sink_wants());
2625 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2626 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2627
2628 // Trigger adapt down, expect limited framerate.
2629 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002630 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002631 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002632 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2634 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2635 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2636
2637 // Trigger adapt down, too small frame, expect no change.
2638 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002639 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002640 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002641 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2642 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2643 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2644 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2645
mflodmancc3d4422017-08-03 08:27:51 -07002646 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002647}
2648
mflodmancc3d4422017-08-03 08:27:51 -07002649TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002650 fake_encoder_.ForceInitEncodeFailure(true);
mflodmancc3d4422017-08-03 08:27:51 -07002651 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002652 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002653 const int kFrameWidth = 1280;
2654 const int kFrameHeight = 720;
2655 video_source_.IncomingCapturedFrame(
2656 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002657 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002658 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002659}
2660
sprangb1ca0732017-02-01 08:38:12 -08002661// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002662TEST_F(VideoStreamEncoderTest,
2663 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
2664 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002665
2666 const int kFrameWidth = 1280;
2667 const int kFrameHeight = 720;
2668 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002669 // requested by
2670 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002671 video_source_.set_adaptation_enabled(true);
2672
2673 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002674 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002675 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002676
2677 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002678 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002679 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002680 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002681 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002682
asaperssonfab67072017-04-04 05:51:49 -07002683 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002684 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002685 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002686 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002687 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002688
mflodmancc3d4422017-08-03 08:27:51 -07002689 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002690}
sprangfe627f32017-03-29 08:24:59 -07002691
mflodmancc3d4422017-08-03 08:27:51 -07002692TEST_F(VideoStreamEncoderTest,
2693 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002694 const int kFrameWidth = 1280;
2695 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002696
mflodmancc3d4422017-08-03 08:27:51 -07002697 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2698 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002699 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002700 video_source_.set_adaptation_enabled(true);
2701
sprang4847ae62017-06-27 07:06:52 -07002702 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002703
2704 video_source_.IncomingCapturedFrame(
2705 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002706 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002707
2708 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002709 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002710
2711 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002712 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002713 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002714 video_source_.IncomingCapturedFrame(
2715 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002716 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002717 }
2718
2719 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002720 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002721 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002722 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002723 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002724 video_source_.IncomingCapturedFrame(
2725 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002726 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002727 ++num_frames_dropped;
2728 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002729 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002730 }
2731 }
2732
sprang4847ae62017-06-27 07:06:52 -07002733 // Add some slack to account for frames dropped by the frame dropper.
2734 const int kErrorMargin = 1;
2735 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002736 kErrorMargin);
2737
2738 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002739 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002740 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002741 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002742 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002743 video_source_.IncomingCapturedFrame(
2744 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002745 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002746 ++num_frames_dropped;
2747 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002748 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002749 }
2750 }
sprang4847ae62017-06-27 07:06:52 -07002751 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002752 kErrorMargin);
2753
2754 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002755 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002756 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002757 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002758 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002759 video_source_.IncomingCapturedFrame(
2760 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002761 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002762 ++num_frames_dropped;
2763 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002764 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002765 }
2766 }
sprang4847ae62017-06-27 07:06:52 -07002767 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002768 kErrorMargin);
2769
2770 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002771 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002772 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002773 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002774 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002775 video_source_.IncomingCapturedFrame(
2776 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002777 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002778 ++num_frames_dropped;
2779 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002780 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002781 }
2782 }
2783 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2784
mflodmancc3d4422017-08-03 08:27:51 -07002785 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002786}
2787
mflodmancc3d4422017-08-03 08:27:51 -07002788TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002789 const int kFramerateFps = 5;
2790 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002791 const int kFrameWidth = 1280;
2792 const int kFrameHeight = 720;
2793
sprang4847ae62017-06-27 07:06:52 -07002794 // Reconfigure encoder with two temporal layers and screensharing, which will
2795 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002796 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002797
mflodmancc3d4422017-08-03 08:27:51 -07002798 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2799 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002800 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002801 video_source_.set_adaptation_enabled(true);
2802
sprang4847ae62017-06-27 07:06:52 -07002803 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002804
2805 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002806 rtc::VideoSinkWants last_wants;
2807 do {
2808 last_wants = video_source_.sink_wants();
2809
sprangc5d62e22017-04-02 23:53:04 -07002810 // Insert frames to get a new fps estimate...
2811 for (int j = 0; j < kFramerateFps; ++j) {
2812 video_source_.IncomingCapturedFrame(
2813 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08002814 if (video_source_.last_sent_width()) {
2815 sink_.WaitForEncodedFrame(timestamp_ms);
2816 }
sprangc5d62e22017-04-02 23:53:04 -07002817 timestamp_ms += kFrameIntervalMs;
Yves Gerey665174f2018-06-19 15:03:05 +02002818 fake_clock_.AdvanceTimeMicros(kFrameIntervalMs *
2819 rtc::kNumMicrosecsPerMillisec);
sprangc5d62e22017-04-02 23:53:04 -07002820 }
2821 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07002822 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08002823 } while (video_source_.sink_wants().max_framerate_fps <
2824 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002825
Jonathan Yubc771b72017-12-08 17:04:29 -08002826 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07002827
mflodmancc3d4422017-08-03 08:27:51 -07002828 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002829}
asaperssonf7e294d2017-06-13 23:25:22 -07002830
mflodmancc3d4422017-08-03 08:27:51 -07002831TEST_F(VideoStreamEncoderTest,
2832 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002833 const int kWidth = 1280;
2834 const int kHeight = 720;
2835 const int64_t kFrameIntervalMs = 150;
2836 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002838
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002839 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002840 AdaptingFrameForwarder source;
2841 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002842 video_stream_encoder_->SetSource(&source,
2843 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002844 timestamp_ms += kFrameIntervalMs;
2845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002846 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002847 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2850 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2851
2852 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002854 timestamp_ms += kFrameIntervalMs;
2855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002856 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002857 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2858 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2859 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2861
2862 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002863 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002864 timestamp_ms += kFrameIntervalMs;
2865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002866 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002867 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2868 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2869 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2870 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2871
2872 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002873 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002874 timestamp_ms += kFrameIntervalMs;
2875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002876 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002877 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2880 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2881
2882 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002883 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002884 timestamp_ms += kFrameIntervalMs;
2885 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002886 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002887 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2888 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2889 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2890 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2891
2892 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002893 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002894 timestamp_ms += kFrameIntervalMs;
2895 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002896 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002897 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2898 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2899 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2900 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2901
2902 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002903 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002904 timestamp_ms += kFrameIntervalMs;
2905 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002906 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002907 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2908 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2909 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2910 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2911
2912 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07002913 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002914 timestamp_ms += kFrameIntervalMs;
2915 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002916 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002917 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2918 rtc::VideoSinkWants last_wants = source.sink_wants();
2919 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2921 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2922
2923 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002924 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002925 timestamp_ms += kFrameIntervalMs;
2926 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002927 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002928 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2929 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2930 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2931 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2932
2933 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002934 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002935 timestamp_ms += kFrameIntervalMs;
2936 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002937 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002938 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2939 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2940 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2941 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2942
2943 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002944 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002945 timestamp_ms += kFrameIntervalMs;
2946 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002947 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002948 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2949 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2950 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2951 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2952
2953 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002954 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002955 timestamp_ms += kFrameIntervalMs;
2956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002957 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002958 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2959 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2960 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2961 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2962
2963 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002964 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002965 timestamp_ms += kFrameIntervalMs;
2966 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002967 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002968 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2969 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2970 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2971 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2972
2973 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002974 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002975 timestamp_ms += kFrameIntervalMs;
2976 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002977 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002978 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
2979 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2980 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2981 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2982
2983 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002985 timestamp_ms += kFrameIntervalMs;
2986 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002987 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002988 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2989 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2990 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2991 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2992
2993 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002994 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002995 timestamp_ms += kFrameIntervalMs;
2996 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002997 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07002998 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002999 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003000 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3002 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3003
3004 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003005 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003006 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003007 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3008
mflodmancc3d4422017-08-03 08:27:51 -07003009 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003010}
3011
mflodmancc3d4422017-08-03 08:27:51 -07003012TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003013 const int kWidth = 1280;
3014 const int kHeight = 720;
3015 const int64_t kFrameIntervalMs = 150;
3016 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003017 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003018
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003019 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003020 AdaptingFrameForwarder source;
3021 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003022 video_stream_encoder_->SetSource(&source,
3023 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003024 timestamp_ms += kFrameIntervalMs;
3025 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003026 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003027 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003028 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3029 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3030 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3031 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3032 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3033 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3034
3035 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003037 timestamp_ms += kFrameIntervalMs;
3038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003039 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003040 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3042 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3043 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3044 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3045 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3046 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3047
3048 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003049 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003050 timestamp_ms += kFrameIntervalMs;
3051 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003052 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003053 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3054 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3055 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3056 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3057 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3058 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3059 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3060
3061 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003062 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003063 timestamp_ms += kFrameIntervalMs;
3064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003065 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003066 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3067 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3068 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3069 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3070 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3071 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3072 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3073
3074 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003075 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003076 timestamp_ms += kFrameIntervalMs;
3077 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003078 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003079 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3080 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3081 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3082 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3083 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3084 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3085 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3086
3087 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003088 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003089 timestamp_ms += kFrameIntervalMs;
3090 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003091 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003092 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3093 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3094 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3095 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3096 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3097 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3098 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3099
3100 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003101 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003102 timestamp_ms += kFrameIntervalMs;
3103 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003104 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003105 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003106 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003107 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3108 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3109 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3110 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3111 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3112 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3113
3114 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003116 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003117 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3118 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3119
mflodmancc3d4422017-08-03 08:27:51 -07003120 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003121}
3122
mflodmancc3d4422017-08-03 08:27:51 -07003123TEST_F(VideoStreamEncoderTest,
3124 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003125 const int kWidth = 640;
3126 const int kHeight = 360;
3127 const int kFpsLimit = 15;
3128 const int64_t kFrameIntervalMs = 150;
3129 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003130 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003131
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003132 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003133 AdaptingFrameForwarder source;
3134 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003135 video_stream_encoder_->SetSource(&source,
3136 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003137 timestamp_ms += kFrameIntervalMs;
3138 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003139 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003140 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003141 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3142 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3143 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3144 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3145 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3146 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3147
3148 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003149 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003150 timestamp_ms += kFrameIntervalMs;
3151 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003152 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003153 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3154 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3155 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3156 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3157 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3158 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3159 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3160
3161 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003163 timestamp_ms += kFrameIntervalMs;
3164 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003165 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003166 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3168 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3169 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3170 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3171 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3172 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3173
3174 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003175 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003176 timestamp_ms += kFrameIntervalMs;
3177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003178 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003179 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3180 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3181 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3182 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3183 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3184 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3185 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3186
3187 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003188 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003189 timestamp_ms += kFrameIntervalMs;
3190 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003191 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003192 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3195 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3196 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3197 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3198 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3199
3200 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003201 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003202 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003203 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3204 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3205
mflodmancc3d4422017-08-03 08:27:51 -07003206 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003207}
3208
mflodmancc3d4422017-08-03 08:27:51 -07003209TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003210 // Simulates simulcast behavior and makes highest stream resolutions divisible
3211 // by 4.
3212 class CroppingVideoStreamFactory
3213 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3214 public:
3215 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3216 int framerate)
3217 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3218 EXPECT_GT(num_temporal_layers, 0u);
3219 EXPECT_GT(framerate, 0);
3220 }
3221
3222 private:
3223 std::vector<VideoStream> CreateEncoderStreams(
3224 int width,
3225 int height,
3226 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003227 std::vector<VideoStream> streams = test::CreateVideoStreams(
3228 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003229 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003230 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003231 stream.max_framerate = framerate_;
3232 }
3233 return streams;
3234 }
3235
3236 const size_t num_temporal_layers_;
3237 const int framerate_;
3238 };
3239
3240 const int kFrameWidth = 1920;
3241 const int kFrameHeight = 1080;
3242 // 3/4 of 1920.
3243 const int kAdaptedFrameWidth = 1440;
3244 // 3/4 of 1080 rounded down to multiple of 4.
3245 const int kAdaptedFrameHeight = 808;
3246 const int kFramerate = 24;
3247
mflodmancc3d4422017-08-03 08:27:51 -07003248 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003249 // Trigger reconfigure encoder (without resetting the entire instance).
3250 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003251 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003252 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3253 video_encoder_config.number_of_streams = 1;
3254 video_encoder_config.video_stream_factory =
3255 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003256 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003257 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003259
3260 video_source_.set_adaptation_enabled(true);
3261
3262 video_source_.IncomingCapturedFrame(
3263 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003264 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003265
3266 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003267 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003268 video_source_.IncomingCapturedFrame(
3269 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003270 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003271
mflodmancc3d4422017-08-03 08:27:51 -07003272 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003273}
3274
mflodmancc3d4422017-08-03 08:27:51 -07003275TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003276 const int kFrameWidth = 1280;
3277 const int kFrameHeight = 720;
3278 const int kLowFps = 2;
3279 const int kHighFps = 30;
3280
mflodmancc3d4422017-08-03 08:27:51 -07003281 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003282
3283 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3284 max_framerate_ = kLowFps;
3285
3286 // Insert 2 seconds of 2fps video.
3287 for (int i = 0; i < kLowFps * 2; ++i) {
3288 video_source_.IncomingCapturedFrame(
3289 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3290 WaitForEncodedFrame(timestamp_ms);
3291 timestamp_ms += 1000 / kLowFps;
3292 }
3293
3294 // Make sure encoder is updated with new target.
mflodmancc3d4422017-08-03 08:27:51 -07003295 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003296 video_source_.IncomingCapturedFrame(
3297 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3298 WaitForEncodedFrame(timestamp_ms);
3299 timestamp_ms += 1000 / kLowFps;
3300
3301 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3302
3303 // Insert 30fps frames for just a little more than the forced update period.
3304 const int kVcmTimerIntervalFrames =
3305 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3306 const int kFrameIntervalMs = 1000 / kHighFps;
3307 max_framerate_ = kHighFps;
3308 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3309 video_source_.IncomingCapturedFrame(
3310 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3311 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3312 // be dropped if the encoder hans't been updated with the new higher target
3313 // framerate yet, causing it to overshoot the target bitrate and then
3314 // suffering the wrath of the media optimizer.
3315 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3316 timestamp_ms += kFrameIntervalMs;
3317 }
3318
3319 // Don expect correct measurement just yet, but it should be higher than
3320 // before.
3321 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3322
mflodmancc3d4422017-08-03 08:27:51 -07003323 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003324}
3325
mflodmancc3d4422017-08-03 08:27:51 -07003326TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003327 const int kFrameWidth = 1280;
3328 const int kFrameHeight = 720;
3329 const int kTargetBitrateBps = 1000000;
3330
3331 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003332 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
mflodmancc3d4422017-08-03 08:27:51 -07003333 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3334 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003335
3336 // Insert a first video frame, causes another bitrate update.
3337 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3338 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3339 video_source_.IncomingCapturedFrame(
3340 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3341 WaitForEncodedFrame(timestamp_ms);
3342
3343 // Next, simulate video suspension due to pacer queue overrun.
mflodmancc3d4422017-08-03 08:27:51 -07003344 video_stream_encoder_->OnBitrateUpdated(0, 0, 1);
sprang4847ae62017-06-27 07:06:52 -07003345
3346 // Skip ahead until a new periodic parameter update should have occured.
3347 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
3348 fake_clock_.AdvanceTimeMicros(
3349 vcm::VCMProcessTimer::kDefaultProcessIntervalMs *
3350 rtc::kNumMicrosecsPerMillisec);
3351
3352 // Bitrate observer should not be called.
3353 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3354 video_source_.IncomingCapturedFrame(
3355 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3356 ExpectDroppedFrame();
3357
mflodmancc3d4422017-08-03 08:27:51 -07003358 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003359}
ilnik6b826ef2017-06-16 06:53:48 -07003360
Niels Möller4db138e2018-04-19 09:04:13 +02003361TEST_F(VideoStreamEncoderTest,
3362 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3363 const int kFrameWidth = 1280;
3364 const int kFrameHeight = 720;
3365 const CpuOveruseOptions default_options;
3366 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3367 video_source_.IncomingCapturedFrame(
3368 CreateFrame(1, kFrameWidth, kFrameHeight));
3369 WaitForEncodedFrame(1);
3370 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3371 .low_encode_usage_threshold_percent,
3372 default_options.low_encode_usage_threshold_percent);
3373 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3374 .high_encode_usage_threshold_percent,
3375 default_options.high_encode_usage_threshold_percent);
3376 video_stream_encoder_->Stop();
3377}
3378
3379TEST_F(VideoStreamEncoderTest,
3380 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3381 const int kFrameWidth = 1280;
3382 const int kFrameHeight = 720;
3383 CpuOveruseOptions hardware_options;
3384 hardware_options.low_encode_usage_threshold_percent = 150;
3385 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003386 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003387
3388 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3389 video_source_.IncomingCapturedFrame(
3390 CreateFrame(1, kFrameWidth, kFrameHeight));
3391 WaitForEncodedFrame(1);
3392 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3393 .low_encode_usage_threshold_percent,
3394 hardware_options.low_encode_usage_threshold_percent);
3395 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3396 .high_encode_usage_threshold_percent,
3397 hardware_options.high_encode_usage_threshold_percent);
3398 video_stream_encoder_->Stop();
3399}
3400
Niels Möller6bb5ab92019-01-11 11:11:10 +01003401TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3402 const int kFrameWidth = 320;
3403 const int kFrameHeight = 240;
3404 const int kFps = 30;
3405 const int kTargetBitrateBps = 120000;
3406 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3407
3408 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3409
3410 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3411 max_framerate_ = kFps;
3412
3413 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3414 fake_encoder_.SimulateOvershoot(1.0);
3415 int num_dropped = 0;
3416 for (int i = 0; i < kNumFramesInRun; ++i) {
3417 video_source_.IncomingCapturedFrame(
3418 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3419 // Wait up to two frame durations for a frame to arrive.
3420 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3421 ++num_dropped;
3422 }
3423 timestamp_ms += 1000 / kFps;
3424 }
3425
Erik Språnga8d48ab2019-02-08 14:17:40 +01003426 // Framerate should be measured to be near the expected target rate.
3427 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3428
3429 // Frame drops should be within 5% of expected 0%.
3430 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003431
3432 // Make encoder produce frames at double the expected bitrate during 3 seconds
3433 // of video, verify number of drops. Rate needs to be slightly changed in
3434 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003435 double overshoot_factor = 2.0;
3436 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3437 // With bitrate adjuster, when need to overshoot even more to trigger
3438 // frame dropping.
3439 overshoot_factor *= 2;
3440 }
3441 fake_encoder_.SimulateOvershoot(overshoot_factor);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003442 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps + 1000, 0, 0);
3443 num_dropped = 0;
3444 for (int i = 0; i < kNumFramesInRun; ++i) {
3445 video_source_.IncomingCapturedFrame(
3446 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3447 // Wait up to two frame durations for a frame to arrive.
3448 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3449 ++num_dropped;
3450 }
3451 timestamp_ms += 1000 / kFps;
3452 }
3453
Erik Språnga8d48ab2019-02-08 14:17:40 +01003454 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3455
3456 // Target framerate should be still be near the expected target, despite
3457 // the frame drops.
3458 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3459
3460 // Frame drops should be within 5% of expected 50%.
3461 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003462
3463 video_stream_encoder_->Stop();
3464}
3465
3466TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3467 const int kFrameWidth = 320;
3468 const int kFrameHeight = 240;
3469 const int kActualInputFps = 24;
3470 const int kTargetBitrateBps = 120000;
3471
3472 ASSERT_GT(max_framerate_, kActualInputFps);
3473
3474 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3475 max_framerate_ = kActualInputFps;
3476 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3477
3478 // Insert 3 seconds of video, with an input fps lower than configured max.
3479 for (int i = 0; i < kActualInputFps * 3; ++i) {
3480 video_source_.IncomingCapturedFrame(
3481 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3482 // Wait up to two frame durations for a frame to arrive.
3483 WaitForEncodedFrame(timestamp_ms);
3484 timestamp_ms += 1000 / kActualInputFps;
3485 }
3486
3487 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3488
3489 video_stream_encoder_->Stop();
3490}
3491
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003492TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3493 VideoFrame::UpdateRect rect;
3494 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3495
3496 fake_encoder_.BlockNextEncode();
3497 video_source_.IncomingCapturedFrame(
3498 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3499 WaitForEncodedFrame(1);
3500 // On the very first frame full update should be forced.
3501 rect = fake_encoder_.GetLastUpdateRect();
3502 EXPECT_EQ(rect.offset_x, 0);
3503 EXPECT_EQ(rect.offset_y, 0);
3504 EXPECT_EQ(rect.height, codec_height_);
3505 EXPECT_EQ(rect.width, codec_width_);
3506 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3507 // call to ContinueEncode.
3508 video_source_.IncomingCapturedFrame(
3509 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3510 ExpectDroppedFrame();
3511 video_source_.IncomingCapturedFrame(
3512 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3513 ExpectDroppedFrame();
3514 fake_encoder_.ContinueEncode();
3515 WaitForEncodedFrame(3);
3516 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3517 rect = fake_encoder_.GetLastUpdateRect();
3518 EXPECT_EQ(rect.offset_x, 1);
3519 EXPECT_EQ(rect.offset_y, 0);
3520 EXPECT_EQ(rect.width, 10);
3521 EXPECT_EQ(rect.height, 1);
3522
3523 video_source_.IncomingCapturedFrame(
3524 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3525 WaitForEncodedFrame(4);
3526 // Previous frame was encoded, so no accumulation should happen.
3527 rect = fake_encoder_.GetLastUpdateRect();
3528 EXPECT_EQ(rect.offset_x, 0);
3529 EXPECT_EQ(rect.offset_y, 0);
3530 EXPECT_EQ(rect.width, 1);
3531 EXPECT_EQ(rect.height, 1);
3532
3533 video_stream_encoder_->Stop();
3534}
3535
Erik Språngd7329ca2019-02-21 21:19:53 +01003536TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
3537 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3538
3539 // First frame is always keyframe.
3540 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3541 WaitForEncodedFrame(1);
3542 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3543 testing::ElementsAre(FrameType{kVideoFrameKey}));
3544
3545 // Insert delta frame.
3546 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3547 WaitForEncodedFrame(2);
3548 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3549 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3550
3551 // Request next frame be a key-frame.
3552 video_stream_encoder_->SendKeyFrame();
3553 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3554 WaitForEncodedFrame(3);
3555 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3556 testing::ElementsAre(FrameType{kVideoFrameKey}));
3557
3558 video_stream_encoder_->Stop();
3559}
3560
3561TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3562 // Setup simulcast with three streams.
3563 ResetEncoder("VP8", 3, 1, 1, false);
3564 video_stream_encoder_->OnBitrateUpdated(kSimulcastTargetBitrateBps, 0, 0);
3565 // Wait for all three layers before triggering event.
3566 sink_.SetNumExpectedLayers(3);
3567
3568 // First frame is always keyframe.
3569 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3570 WaitForEncodedFrame(1);
3571 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3572 testing::ElementsAreArray(
3573 {kVideoFrameKey, kVideoFrameKey, kVideoFrameKey}));
3574
3575 // Insert delta frame.
3576 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3577 WaitForEncodedFrame(2);
3578 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3579 testing::ElementsAreArray(
3580 {kVideoFrameDelta, kVideoFrameDelta, kVideoFrameDelta}));
3581
3582 // Request next frame be a key-frame.
3583 // Only first stream is configured to produce key-frame.
3584 video_stream_encoder_->SendKeyFrame();
3585 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3586 WaitForEncodedFrame(3);
3587 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3588 testing::ElementsAreArray(
3589 {kVideoFrameKey, kVideoFrameDelta, kVideoFrameDelta}));
3590
3591 video_stream_encoder_->Stop();
3592}
3593
3594TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3595 // Configure internal source factory and setup test again.
3596 encoder_factory_.SetHasInternalSource(true);
3597 ResetEncoder("VP8", 1, 1, 1, false);
3598 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3599
3600 // Call encoder directly, simulating internal source where encoded frame
3601 // callback in VideoStreamEncoder is called despite no OnFrame().
3602 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3603 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3604 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3605 testing::ElementsAre(FrameType{kVideoFrameKey}));
3606
3607 const std::vector<FrameType> kDeltaFrame = {kVideoFrameDelta};
3608 // Need to set timestamp manually since manually for injected frame.
3609 VideoFrame frame = CreateFrame(101, nullptr);
3610 frame.set_timestamp(101);
3611 fake_encoder_.InjectFrame(frame, false);
3612 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3613 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3614 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3615
3616 // Request key-frame. The forces a dummy frame down into the encoder.
3617 fake_encoder_.ExpectNullFrame();
3618 video_stream_encoder_->SendKeyFrame();
3619 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3620 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3621 testing::ElementsAre(FrameType{kVideoFrameKey}));
3622
3623 video_stream_encoder_->Stop();
3624}
Erik Språngb7cb7b52019-02-26 15:52:33 +01003625
3626TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
3627 // Configure internal source factory and setup test again.
3628 encoder_factory_.SetHasInternalSource(true);
3629 ResetEncoder("VP8", 1, 1, 1, false);
3630 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3631
3632 int64_t timestamp = 1;
3633 EncodedImage image;
3634 image.Allocate(kTargetBitrateBps / kDefaultFramerate / 8);
3635 image.capture_time_ms_ = ++timestamp;
3636 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
3637 const int64_t kEncodeFinishDelayMs = 10;
3638 image.timing_.encode_start_ms = timestamp;
3639 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
3640 fake_encoder_.InjectEncodedImage(image);
3641 // Wait for frame without incrementing clock.
3642 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3643 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
3644 // capture timestamp should be kEncodeFinishDelayMs in the past.
3645 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
3646 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
3647 kEncodeFinishDelayMs);
3648
3649 video_stream_encoder_->Stop();
3650}
perkj26091b12016-09-01 01:17:40 -07003651} // namespace webrtc