blob: 75b47f28033d4356141fd555bdadbf6330c72865 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Per512ecb32016-09-23 15:52:06 +020015#include <utility>
16
Sebastian Jansson74682c12019-03-01 11:50:20 +010017#include "api/task_queue/global_task_queue_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080018#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010020#include "api/video/video_bitrate_allocation.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020021#include "api/video_codecs/create_vp8_temporal_layers.h"
22#include "api/video_codecs/vp8_temporal_layers.h"
Steve Anton10542f22019-01-11 09:11:00 -080023#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020024#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080026#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080028#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010029#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020030#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "system_wrappers/include/sleep.h"
32#include "test/encoder_settings.h"
33#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020034#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "test/frame_generator.h"
36#include "test/gmock.h"
37#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020038#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070040
41namespace webrtc {
42
sprangb1ca0732017-02-01 08:38:12 -080043using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080044using ::testing::_;
45using ::testing::Return;
kthelgason876222f2016-11-29 01:44:11 -080046
perkj803d97f2016-11-01 11:45:46 -070047namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020048const int kMinPixelsPerFrame = 320 * 180;
49const int kMinFramerateFps = 2;
50const int kMinBalancedFramerateFps = 7;
51const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080052const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010053const uint32_t kTargetBitrateBps = 1000000;
54const uint32_t kSimulcastTargetBitrateBps = 3150000;
55const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080056const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070057const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020058const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080059
perkj803d97f2016-11-01 11:45:46 -070060class TestBuffer : public webrtc::I420Buffer {
61 public:
62 TestBuffer(rtc::Event* event, int width, int height)
63 : I420Buffer(width, height), event_(event) {}
64
65 private:
66 friend class rtc::RefCountedObject<TestBuffer>;
67 ~TestBuffer() override {
68 if (event_)
69 event_->Set();
70 }
71 rtc::Event* const event_;
72};
73
Niels Möller7dc26b72017-12-06 10:27:48 +010074class CpuOveruseDetectorProxy : public OveruseFrameDetector {
75 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +020076 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
77 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +010078 last_target_framerate_fps_(-1) {}
79 virtual ~CpuOveruseDetectorProxy() {}
80
81 void OnTargetFramerateUpdated(int framerate_fps) override {
82 rtc::CritScope cs(&lock_);
83 last_target_framerate_fps_ = framerate_fps;
84 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
85 }
86
87 int GetLastTargetFramerate() {
88 rtc::CritScope cs(&lock_);
89 return last_target_framerate_fps_;
90 }
91
Niels Möller4db138e2018-04-19 09:04:13 +020092 CpuOveruseOptions GetOptions() { return options_; }
93
Niels Möller7dc26b72017-12-06 10:27:48 +010094 private:
95 rtc::CriticalSection lock_;
96 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
97};
98
mflodmancc3d4422017-08-03 08:27:51 -070099class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700100 public:
Niels Möller213618e2018-07-24 09:29:58 +0200101 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
102 const VideoStreamEncoderSettings& settings)
Yves Gerey665174f2018-06-19 15:03:05 +0200103 : VideoStreamEncoder(1 /* number_of_cores */,
104 stats_proxy,
105 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200106 std::unique_ptr<OveruseFrameDetector>(
107 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100108 new CpuOveruseDetectorProxy(stats_proxy)),
109 &GlobalTaskQueueFactory()) {}
perkj803d97f2016-11-01 11:45:46 -0700110
sprangb1ca0732017-02-01 08:38:12 -0800111 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100112 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800113 encoder_queue()->PostTask([this, &event, reason, down] {
sprangb1ca0732017-02-01 08:38:12 -0800114 down ? AdaptDown(reason) : AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700115 event.Set();
116 });
perkj070ba852017-02-16 15:46:27 -0800117 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700118 }
119
kthelgason2fc52542017-03-03 00:24:41 -0800120 // This is used as a synchronisation mechanism, to make sure that the
121 // encoder queue is not blocked before we start sending it frames.
122 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100123 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200124 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800125 ASSERT_TRUE(event.Wait(5000));
126 }
127
sprangb1ca0732017-02-01 08:38:12 -0800128 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800129
sprangb1ca0732017-02-01 08:38:12 -0800130 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800131
sprangb1ca0732017-02-01 08:38:12 -0800132 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800133
sprangb1ca0732017-02-01 08:38:12 -0800134 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700135
Niels Möller7dc26b72017-12-06 10:27:48 +0100136 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700137};
138
asapersson5f7226f2016-11-25 04:37:00 -0800139class VideoStreamFactory
140 : public VideoEncoderConfig::VideoStreamFactoryInterface {
141 public:
sprangfda496a2017-06-15 04:21:07 -0700142 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
143 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800144 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700145 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800146 }
147
148 private:
149 std::vector<VideoStream> CreateEncoderStreams(
150 int width,
151 int height,
152 const VideoEncoderConfig& encoder_config) override {
153 std::vector<VideoStream> streams =
154 test::CreateVideoStreams(width, height, encoder_config);
155 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100156 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700157 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800158 }
159 return streams;
160 }
sprangfda496a2017-06-15 04:21:07 -0700161
asapersson5f7226f2016-11-25 04:37:00 -0800162 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700163 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800164};
165
sprangb1ca0732017-02-01 08:38:12 -0800166class AdaptingFrameForwarder : public test::FrameForwarder {
167 public:
168 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700169 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800170
171 void set_adaptation_enabled(bool enabled) {
172 rtc::CritScope cs(&crit_);
173 adaptation_enabled_ = enabled;
174 }
175
asaperssonfab67072017-04-04 05:51:49 -0700176 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800177 rtc::CritScope cs(&crit_);
178 return adaptation_enabled_;
179 }
180
asapersson09f05612017-05-15 23:40:18 -0700181 rtc::VideoSinkWants last_wants() const {
182 rtc::CritScope cs(&crit_);
183 return last_wants_;
184 }
185
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200186 absl::optional<int> last_sent_width() const { return last_width_; }
187 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800188
sprangb1ca0732017-02-01 08:38:12 -0800189 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
190 int cropped_width = 0;
191 int cropped_height = 0;
192 int out_width = 0;
193 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700194 if (adaption_enabled()) {
195 if (adapter_.AdaptFrameResolution(
196 video_frame.width(), video_frame.height(),
197 video_frame.timestamp_us() * 1000, &cropped_width,
198 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100199 VideoFrame adapted_frame =
200 VideoFrame::Builder()
201 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
202 nullptr, out_width, out_height))
203 .set_timestamp_rtp(99)
204 .set_timestamp_ms(99)
205 .set_rotation(kVideoRotation_0)
206 .build();
sprangc5d62e22017-04-02 23:53:04 -0700207 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
208 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800209 last_width_.emplace(adapted_frame.width());
210 last_height_.emplace(adapted_frame.height());
211 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200212 last_width_ = absl::nullopt;
213 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700214 }
sprangb1ca0732017-02-01 08:38:12 -0800215 } else {
216 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800217 last_width_.emplace(video_frame.width());
218 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800219 }
220 }
221
222 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
223 const rtc::VideoSinkWants& wants) override {
224 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700225 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700226 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
227 wants.max_pixel_count,
228 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800229 test::FrameForwarder::AddOrUpdateSink(sink, wants);
230 }
sprangb1ca0732017-02-01 08:38:12 -0800231 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700232 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
233 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200234 absl::optional<int> last_width_;
235 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800236};
sprangc5d62e22017-04-02 23:53:04 -0700237
Niels Möller213618e2018-07-24 09:29:58 +0200238// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700239class MockableSendStatisticsProxy : public SendStatisticsProxy {
240 public:
241 MockableSendStatisticsProxy(Clock* clock,
242 const VideoSendStream::Config& config,
243 VideoEncoderConfig::ContentType content_type)
244 : SendStatisticsProxy(clock, config, content_type) {}
245
246 VideoSendStream::Stats GetStats() override {
247 rtc::CritScope cs(&lock_);
248 if (mock_stats_)
249 return *mock_stats_;
250 return SendStatisticsProxy::GetStats();
251 }
252
Niels Möller213618e2018-07-24 09:29:58 +0200253 int GetInputFrameRate() const override {
254 rtc::CritScope cs(&lock_);
255 if (mock_stats_)
256 return mock_stats_->input_frame_rate;
257 return SendStatisticsProxy::GetInputFrameRate();
258 }
sprangc5d62e22017-04-02 23:53:04 -0700259 void SetMockStats(const VideoSendStream::Stats& stats) {
260 rtc::CritScope cs(&lock_);
261 mock_stats_.emplace(stats);
262 }
263
264 void ResetMockStats() {
265 rtc::CritScope cs(&lock_);
266 mock_stats_.reset();
267 }
268
269 private:
270 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200271 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700272};
273
sprang4847ae62017-06-27 07:06:52 -0700274class MockBitrateObserver : public VideoBitrateAllocationObserver {
275 public:
Erik Språng566124a2018-04-23 12:32:22 +0200276 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700277};
278
perkj803d97f2016-11-01 11:45:46 -0700279} // namespace
280
mflodmancc3d4422017-08-03 08:27:51 -0700281class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700282 public:
283 static const int kDefaultTimeoutMs = 30 * 1000;
284
mflodmancc3d4422017-08-03 08:27:51 -0700285 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700286 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700287 codec_width_(320),
288 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200289 max_framerate_(kDefaultFramerate),
perkj26091b12016-09-01 01:17:40 -0700290 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200291 encoder_factory_(&fake_encoder_),
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800292 bitrate_allocator_factory_(CreateBuiltinVideoBitrateAllocatorFactory()),
sprangc5d62e22017-04-02 23:53:04 -0700293 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700294 Clock::GetRealTimeClock(),
295 video_send_config_,
296 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700297 sink_(&fake_encoder_) {}
298
299 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700300 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700301 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200302 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800303 video_send_config_.encoder_settings.bitrate_allocator_factory =
304 bitrate_allocator_factory_.get();
Niels Möller259a4972018-04-05 15:36:51 +0200305 video_send_config_.rtp.payload_name = "FAKE";
306 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700307
Per512ecb32016-09-23 15:52:06 +0200308 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200309 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700310 video_encoder_config.video_stream_factory =
311 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100312 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700313
314 // Framerate limit is specified by the VideoStreamFactory.
315 std::vector<VideoStream> streams =
316 video_encoder_config.video_stream_factory->CreateEncoderStreams(
317 codec_width_, codec_height_, video_encoder_config);
318 max_framerate_ = streams[0].max_framerate;
319 fake_clock_.SetTimeMicros(1234);
320
Niels Möllerf1338562018-04-26 09:51:47 +0200321 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800322 }
323
Niels Möllerf1338562018-04-26 09:51:47 +0200324 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700325 if (video_stream_encoder_)
326 video_stream_encoder_->Stop();
327 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
perkj803d97f2016-11-01 11:45:46 -0700328 stats_proxy_.get(), video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700329 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
330 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700331 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700332 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
333 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200334 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700335 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800336 }
337
338 void ResetEncoder(const std::string& payload_name,
339 size_t num_streams,
340 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700341 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700342 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200343 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800344
345 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200346 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800347 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100348 video_encoder_config.max_bitrate_bps =
349 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800350 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700351 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
352 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700353 video_encoder_config.content_type =
354 screenshare ? VideoEncoderConfig::ContentType::kScreen
355 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700356 if (payload_name == "VP9") {
357 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
358 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
359 video_encoder_config.encoder_specific_settings =
360 new rtc::RefCountedObject<
361 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
362 }
Niels Möllerf1338562018-04-26 09:51:47 +0200363 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700364 }
365
sprang57c2fff2017-01-16 06:24:02 -0800366 VideoFrame CreateFrame(int64_t ntp_time_ms,
367 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100368 VideoFrame frame =
369 VideoFrame::Builder()
370 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
371 destruction_event, codec_width_, codec_height_))
372 .set_timestamp_rtp(99)
373 .set_timestamp_ms(99)
374 .set_rotation(kVideoRotation_0)
375 .build();
sprang57c2fff2017-01-16 06:24:02 -0800376 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700377 return frame;
378 }
379
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100380 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
381 rtc::Event* destruction_event,
382 int offset_x) const {
383 VideoFrame frame =
384 VideoFrame::Builder()
385 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
386 destruction_event, codec_width_, codec_height_))
387 .set_timestamp_rtp(99)
388 .set_timestamp_ms(99)
389 .set_rotation(kVideoRotation_0)
390 .set_update_rect({offset_x, 0, 1, 1})
391 .build();
392 frame.set_ntp_time_ms(ntp_time_ms);
393 return frame;
394 }
395
sprang57c2fff2017-01-16 06:24:02 -0800396 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100397 VideoFrame frame =
398 VideoFrame::Builder()
399 .set_video_frame_buffer(
400 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
401 .set_timestamp_rtp(99)
402 .set_timestamp_ms(99)
403 .set_rotation(kVideoRotation_0)
404 .build();
sprang57c2fff2017-01-16 06:24:02 -0800405 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700406 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700407 return frame;
408 }
409
asapersson02465b82017-04-10 01:12:52 -0700410 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700411 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700412 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
413 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700414 }
415
asapersson09f05612017-05-15 23:40:18 -0700416 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
417 const rtc::VideoSinkWants& wants2) {
418 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
419 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
420 }
421
Åsa Persson8c1bf952018-09-13 10:42:19 +0200422 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
423 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
424 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
425 EXPECT_FALSE(wants.target_pixel_count);
426 }
427
asapersson09f05612017-05-15 23:40:18 -0700428 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
429 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200430 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700431 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
432 EXPECT_GT(wants1.max_pixel_count, 0);
433 }
434
435 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
436 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200437 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700438 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
439 }
440
asaperssonf7e294d2017-06-13 23:25:22 -0700441 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
442 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200443 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700444 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
445 }
446
447 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
448 const rtc::VideoSinkWants& wants2) {
449 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
450 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
451 }
452
453 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
454 const rtc::VideoSinkWants& wants2) {
455 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
456 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
457 }
458
459 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
460 const rtc::VideoSinkWants& wants2) {
461 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
462 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
463 EXPECT_GT(wants1.max_pixel_count, 0);
464 }
465
466 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
467 const rtc::VideoSinkWants& wants2) {
468 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
469 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
470 }
471
asapersson09f05612017-05-15 23:40:18 -0700472 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
473 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200474 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700475 EXPECT_LT(wants.max_pixel_count, pixel_count);
476 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700477 }
478
479 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
480 EXPECT_LT(wants.max_framerate_fps, fps);
481 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
482 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700483 }
484
asaperssonf7e294d2017-06-13 23:25:22 -0700485 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
486 int expected_fps) {
487 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
488 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
489 EXPECT_FALSE(wants.target_pixel_count);
490 }
491
Jonathan Yubc771b72017-12-08 17:04:29 -0800492 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
493 int last_frame_pixels) {
494 // Balanced mode should always scale FPS to the desired range before
495 // attempting to scale resolution.
496 int fps_limit = wants.max_framerate_fps;
497 if (last_frame_pixels <= 320 * 240) {
498 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
499 } else if (last_frame_pixels <= 480 * 270) {
500 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
501 } else if (last_frame_pixels <= 640 * 480) {
502 EXPECT_LE(15, fps_limit);
503 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200504 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800505 }
506 }
507
sprang4847ae62017-06-27 07:06:52 -0700508 void WaitForEncodedFrame(int64_t expected_ntp_time) {
509 sink_.WaitForEncodedFrame(expected_ntp_time);
510 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
511 }
512
513 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
514 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
515 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
516 return ok;
517 }
518
519 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
520 sink_.WaitForEncodedFrame(expected_width, expected_height);
521 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
522 }
523
524 void ExpectDroppedFrame() {
525 sink_.ExpectDroppedFrame();
526 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
527 }
528
529 bool WaitForFrame(int64_t timeout_ms) {
530 bool ok = sink_.WaitForFrame(timeout_ms);
531 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerSec / max_framerate_);
532 return ok;
533 }
534
perkj26091b12016-09-01 01:17:40 -0700535 class TestEncoder : public test::FakeEncoder {
536 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100537 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700538
asaperssonfab67072017-04-04 05:51:49 -0700539 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800540 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700541 return config_;
542 }
543
544 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800545 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700546 block_next_encode_ = true;
547 }
548
Erik Språngaed30702018-11-05 12:57:17 +0100549 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800550 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100551 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100552 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100553 if (quality_scaling_) {
554 info.scaling_settings =
555 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
556 }
557 info.is_hardware_accelerated = is_hardware_accelerated_;
Erik Språngaed30702018-11-05 12:57:17 +0100558 }
559 return info;
kthelgason876222f2016-11-29 01:44:11 -0800560 }
561
Erik Språngb7cb7b52019-02-26 15:52:33 +0100562 int32_t RegisterEncodeCompleteCallback(
563 EncodedImageCallback* callback) override {
564 rtc::CritScope lock(&local_crit_sect_);
565 encoded_image_callback_ = callback;
566 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
567 }
568
perkjfa10b552016-10-02 23:45:26 -0700569 void ContinueEncode() { continue_encode_event_.Set(); }
570
571 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
572 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800573 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700574 EXPECT_EQ(timestamp_, timestamp);
575 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
576 }
577
kthelgason2fc52542017-03-03 00:24:41 -0800578 void SetQualityScaling(bool b) {
579 rtc::CritScope lock(&local_crit_sect_);
580 quality_scaling_ = b;
581 }
kthelgasonad9010c2017-02-14 00:46:51 -0800582
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100583 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
584 rtc::CritScope lock(&local_crit_sect_);
585 is_hardware_accelerated_ = is_hardware_accelerated;
586 }
587
sprangfe627f32017-03-29 08:24:59 -0700588 void ForceInitEncodeFailure(bool force_failure) {
589 rtc::CritScope lock(&local_crit_sect_);
590 force_init_encode_failed_ = force_failure;
591 }
592
Niels Möller6bb5ab92019-01-11 11:11:10 +0100593 void SimulateOvershoot(double rate_factor) {
594 rtc::CritScope lock(&local_crit_sect_);
595 rate_factor_ = rate_factor;
596 }
597
Erik Språngd7329ca2019-02-21 21:19:53 +0100598 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100599 rtc::CritScope lock(&local_crit_sect_);
600 return last_framerate_;
601 }
602
Erik Språngd7329ca2019-02-21 21:19:53 +0100603 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100604 rtc::CritScope lock(&local_crit_sect_);
605 return last_update_rect_;
606 }
607
Erik Språngd7329ca2019-02-21 21:19:53 +0100608 const std::vector<FrameType>& LastFrameTypes() const {
609 rtc::CritScope lock(&local_crit_sect_);
610 return last_frame_types_;
611 }
612
613 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
614 const std::vector<FrameType> frame_type = {keyframe ? kVideoFrameKey
615 : kVideoFrameDelta};
616 {
617 rtc::CritScope lock(&local_crit_sect_);
618 last_frame_types_ = frame_type;
619 }
620 FakeEncoder::Encode(input_image, nullptr, &frame_type);
621 }
622
Erik Språngb7cb7b52019-02-26 15:52:33 +0100623 void InjectEncodedImage(const EncodedImage& image) {
624 rtc::CritScope lock(&local_crit_sect_);
625 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
626 }
627
Erik Språngd7329ca2019-02-21 21:19:53 +0100628 void ExpectNullFrame() {
629 rtc::CritScope lock(&local_crit_sect_);
630 expect_null_frame_ = true;
631 }
632
633 absl::optional<VideoBitrateAllocation> GetAndResetLastBitrateAllocation() {
634 auto allocation = last_bitrate_allocation_;
635 last_bitrate_allocation_.reset();
636 return allocation;
637 }
638
perkjfa10b552016-10-02 23:45:26 -0700639 private:
perkj26091b12016-09-01 01:17:40 -0700640 int32_t Encode(const VideoFrame& input_image,
641 const CodecSpecificInfo* codec_specific_info,
642 const std::vector<FrameType>* frame_types) override {
643 bool block_encode;
644 {
brandtre78d2662017-01-16 05:57:16 -0800645 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100646 if (expect_null_frame_) {
647 EXPECT_EQ(input_image.timestamp(), 0u);
648 EXPECT_EQ(input_image.width(), 1);
649 last_frame_types_ = *frame_types;
650 expect_null_frame_ = false;
651 } else {
652 EXPECT_GT(input_image.timestamp(), timestamp_);
653 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
654 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
655 }
perkj26091b12016-09-01 01:17:40 -0700656
657 timestamp_ = input_image.timestamp();
658 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700659 last_input_width_ = input_image.width();
660 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700661 block_encode = block_next_encode_;
662 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100663 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100664 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700665 }
666 int32_t result =
667 FakeEncoder::Encode(input_image, codec_specific_info, frame_types);
668 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700669 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700670 return result;
671 }
672
sprangfe627f32017-03-29 08:24:59 -0700673 int32_t InitEncode(const VideoCodec* config,
674 int32_t number_of_cores,
675 size_t max_payload_size) override {
676 int res =
677 FakeEncoder::InitEncode(config, number_of_cores, max_payload_size);
678 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100679 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Erik Språng82fad3d2018-03-21 09:57:23 +0100680 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700681 // Simulate setting up temporal layers, in order to validate the life
682 // cycle of these objects.
683 int num_streams = std::max<int>(1, config->numberOfSimulcastStreams);
sprangfe627f32017-03-29 08:24:59 -0700684 for (int i = 0; i < num_streams; ++i) {
685 allocated_temporal_layers_.emplace_back(
Erik Språng4529fbc2018-10-12 10:30:31 +0200686 CreateVp8TemporalLayers(Vp8TemporalLayersType::kFixedPattern,
687 config->VP8().numberOfTemporalLayers));
sprangfe627f32017-03-29 08:24:59 -0700688 }
689 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100690 if (force_init_encode_failed_) {
691 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700692 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100693 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100694
Erik Språngb7cb7b52019-02-26 15:52:33 +0100695 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700696 return res;
697 }
698
Erik Språngb7cb7b52019-02-26 15:52:33 +0100699 int32_t Release() override {
700 rtc::CritScope lock(&local_crit_sect_);
701 EXPECT_NE(initialized_, EncoderState::kUninitialized);
702 initialized_ = EncoderState::kUninitialized;
703 return FakeEncoder::Release();
704 }
705
Niels Möller6bb5ab92019-01-11 11:11:10 +0100706 int32_t SetRateAllocation(const VideoBitrateAllocation& rate_allocation,
707 uint32_t framerate) {
708 rtc::CritScope lock(&local_crit_sect_);
709 VideoBitrateAllocation adjusted_rate_allocation;
710 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
711 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
712 if (rate_allocation.HasBitrate(si, ti)) {
713 adjusted_rate_allocation.SetBitrate(
714 si, ti,
715 static_cast<uint32_t>(rate_allocation.GetBitrate(si, ti) *
716 rate_factor_));
717 }
718 }
719 }
720 last_framerate_ = framerate;
Erik Språngd7329ca2019-02-21 21:19:53 +0100721 last_bitrate_allocation_ = rate_allocation;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100722 return FakeEncoder::SetRateAllocation(adjusted_rate_allocation,
723 framerate);
724 }
725
brandtre78d2662017-01-16 05:57:16 -0800726 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100727 enum class EncoderState {
728 kUninitialized,
729 kInitializationFailed,
730 kInitialized
731 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
732 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700733 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700734 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700735 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
736 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
737 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
738 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
739 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100740 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Erik Språng4529fbc2018-10-12 10:30:31 +0200741 std::vector<std::unique_ptr<Vp8TemporalLayers>> allocated_temporal_layers_
danilchapa37de392017-09-09 04:17:22 -0700742 RTC_GUARDED_BY(local_crit_sect_);
743 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100744 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
745 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100746 absl::optional<VideoBitrateAllocation> last_bitrate_allocation_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100747 VideoFrame::UpdateRect last_update_rect_
748 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Erik Språngd7329ca2019-02-21 21:19:53 +0100749 std::vector<FrameType> last_frame_types_;
750 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100751 EncodedImageCallback* encoded_image_callback_
752 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
perkj26091b12016-09-01 01:17:40 -0700753 };
754
mflodmancc3d4422017-08-03 08:27:51 -0700755 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700756 public:
757 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100758 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700759
perkj26091b12016-09-01 01:17:40 -0700760 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700761 EXPECT_TRUE(
762 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
763 }
764
765 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
766 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700767 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700768 if (!encoded_frame_event_.Wait(timeout_ms))
769 return false;
perkj26091b12016-09-01 01:17:40 -0700770 {
771 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800772 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700773 }
774 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700775 return true;
perkj26091b12016-09-01 01:17:40 -0700776 }
777
sprangb1ca0732017-02-01 08:38:12 -0800778 void WaitForEncodedFrame(uint32_t expected_width,
779 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700780 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100781 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700782 }
783
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100784 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700785 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800786 uint32_t width = 0;
787 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800788 {
789 rtc::CritScope lock(&crit_);
790 width = last_width_;
791 height = last_height_;
792 }
793 EXPECT_EQ(expected_height, height);
794 EXPECT_EQ(expected_width, width);
795 }
796
kthelgason2fc52542017-03-03 00:24:41 -0800797 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800798
sprangc5d62e22017-04-02 23:53:04 -0700799 bool WaitForFrame(int64_t timeout_ms) {
800 return encoded_frame_event_.Wait(timeout_ms);
801 }
802
perkj26091b12016-09-01 01:17:40 -0700803 void SetExpectNoFrames() {
804 rtc::CritScope lock(&crit_);
805 expect_frames_ = false;
806 }
807
asaperssonfab67072017-04-04 05:51:49 -0700808 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200809 rtc::CritScope lock(&crit_);
810 return number_of_reconfigurations_;
811 }
812
asaperssonfab67072017-04-04 05:51:49 -0700813 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200814 rtc::CritScope lock(&crit_);
815 return min_transmit_bitrate_bps_;
816 }
817
Erik Språngd7329ca2019-02-21 21:19:53 +0100818 void SetNumExpectedLayers(size_t num_layers) {
819 rtc::CritScope lock(&crit_);
820 num_expected_layers_ = num_layers;
821 }
822
Erik Språngb7cb7b52019-02-26 15:52:33 +0100823 int64_t GetLastCaptureTimeMs() const {
824 rtc::CritScope lock(&crit_);
825 return last_capture_time_ms_;
826 }
827
perkj26091b12016-09-01 01:17:40 -0700828 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700829 Result OnEncodedImage(
830 const EncodedImage& encoded_image,
831 const CodecSpecificInfo* codec_specific_info,
832 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200833 rtc::CritScope lock(&crit_);
834 EXPECT_TRUE(expect_frames_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100835 uint32_t timestamp = encoded_image.Timestamp();
836 if (last_timestamp_ != timestamp) {
837 num_received_layers_ = 1;
838 } else {
839 ++num_received_layers_;
840 }
841 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100842 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -0800843 last_width_ = encoded_image._encodedWidth;
844 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +0100845 if (num_received_layers_ == num_expected_layers_) {
846 encoded_frame_event_.Set();
847 }
sprangb1ca0732017-02-01 08:38:12 -0800848 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +0200849 }
850
Rasmus Brandtc402dbe2019-02-04 11:09:46 +0100851 void OnEncoderConfigurationChanged(
852 std::vector<VideoStream> streams,
853 VideoEncoderConfig::ContentType content_type,
854 int min_transmit_bitrate_bps) override {
Per512ecb32016-09-23 15:52:06 +0200855 rtc::CriticalSection crit_;
856 ++number_of_reconfigurations_;
857 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
858 }
859
perkj26091b12016-09-01 01:17:40 -0700860 rtc::CriticalSection crit_;
861 TestEncoder* test_encoder_;
862 rtc::Event encoded_frame_event_;
sprangb1ca0732017-02-01 08:38:12 -0800863 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100864 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -0800865 uint32_t last_height_ = 0;
866 uint32_t last_width_ = 0;
Erik Språngd7329ca2019-02-21 21:19:53 +0100867 size_t num_expected_layers_ = 1;
868 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -0700869 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +0200870 int number_of_reconfigurations_ = 0;
871 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -0700872 };
873
874 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +0100875 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +0200876 int codec_width_;
877 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -0700878 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -0700879 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +0200880 test::VideoEncoderProxyFactory encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800881 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -0700882 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -0700883 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -0800884 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -0700885 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
sprang4847ae62017-06-27 07:06:52 -0700886 rtc::ScopedFakeClock fake_clock_;
perkj26091b12016-09-01 01:17:40 -0700887};
888
mflodmancc3d4422017-08-03 08:27:51 -0700889TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
890 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +0100891 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700892 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -0700893 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -0700894 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -0700895 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700896}
897
mflodmancc3d4422017-08-03 08:27:51 -0700898TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -0700899 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +0100900 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +0200901 // The encoder will cache up to one frame for a short duration. Adding two
902 // frames means that the first frame will be dropped and the second frame will
903 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -0700904 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +0200905 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -0700906 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700907
mflodmancc3d4422017-08-03 08:27:51 -0700908 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700909
Sebastian Janssona3177052018-04-10 13:05:49 +0200910 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -0700911 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +0200912 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
913
914 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -0700915 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700916}
917
mflodmancc3d4422017-08-03 08:27:51 -0700918TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
919 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700920 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700921 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700922
mflodmancc3d4422017-08-03 08:27:51 -0700923 video_stream_encoder_->OnBitrateUpdated(0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +0200924 // The encoder will cache up to one frame for a short duration. Adding two
925 // frames means that the first frame will be dropped and the second frame will
926 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -0700927 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +0200928 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700929
mflodmancc3d4422017-08-03 08:27:51 -0700930 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -0700931 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +0200932 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
933 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -0700934 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700935}
936
mflodmancc3d4422017-08-03 08:27:51 -0700937TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
938 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkja49cbd32016-09-16 07:53:41 -0700939 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700940 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700941
942 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -0700943 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -0700944
perkja49cbd32016-09-16 07:53:41 -0700945 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700946 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -0700947 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700948}
949
mflodmancc3d4422017-08-03 08:27:51 -0700950TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
951 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700952
perkja49cbd32016-09-16 07:53:41 -0700953 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700954 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700955
mflodmancc3d4422017-08-03 08:27:51 -0700956 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700957 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +0100958 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -0700959 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
960 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700961}
962
mflodmancc3d4422017-08-03 08:27:51 -0700963TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
964 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj26091b12016-09-01 01:17:40 -0700965
966 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -0700967 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700968 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -0700969 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
970 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -0700971 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
972 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -0700973 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -0700974 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -0700975
mflodmancc3d4422017-08-03 08:27:51 -0700976 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -0700977}
978
mflodmancc3d4422017-08-03 08:27:51 -0700979TEST_F(VideoStreamEncoderTest,
980 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
981 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Per21d45d22016-10-30 21:37:57 +0100982 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200983
984 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200985 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700986 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +0100987 // The encoder will have been configured once when the first frame is
988 // received.
989 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +0200990
991 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200992 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +0200993 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -0700994 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200995 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +0200996
997 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +0200998 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -0700999 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001000 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001001 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001002
mflodmancc3d4422017-08-03 08:27:51 -07001003 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001004}
1005
mflodmancc3d4422017-08-03 08:27:51 -07001006TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
1007 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001008
1009 // Capture a frame and wait for it to synchronize with the encoder thread.
1010 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001011 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001012 // The encoder will have been configured once.
1013 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001014 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1015 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1016
1017 codec_width_ *= 2;
1018 codec_height_ *= 2;
1019 // Capture a frame with a higher resolution and wait for it to synchronize
1020 // with the encoder thread.
1021 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001022 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001023 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1024 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001025 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001026
mflodmancc3d4422017-08-03 08:27:51 -07001027 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001028}
1029
mflodmancc3d4422017-08-03 08:27:51 -07001030TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001031 EXPECT_TRUE(video_source_.has_sinks());
1032 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001033 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001034 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001035 EXPECT_FALSE(video_source_.has_sinks());
1036 EXPECT_TRUE(new_video_source.has_sinks());
1037
mflodmancc3d4422017-08-03 08:27:51 -07001038 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001039}
1040
mflodmancc3d4422017-08-03 08:27:51 -07001041TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001042 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001043 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001044 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001045 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001046}
1047
Jonathan Yubc771b72017-12-08 17:04:29 -08001048TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1049 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001050 const int kWidth = 1280;
1051 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001052
1053 // We rely on the automatic resolution adaptation, but we handle framerate
1054 // adaptation manually by mocking the stats proxy.
1055 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001056
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001057 // Enable BALANCED preference, no initial limitation.
Jonathan Yubc771b72017-12-08 17:04:29 -08001058 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001059 video_stream_encoder_->SetSource(&video_source_,
1060 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001061 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001062 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001063 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001064 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1065
Jonathan Yubc771b72017-12-08 17:04:29 -08001066 // Adapt down as far as possible.
1067 rtc::VideoSinkWants last_wants;
1068 int64_t t = 1;
1069 int loop_count = 0;
1070 do {
1071 ++loop_count;
1072 last_wants = video_source_.sink_wants();
1073
1074 // Simulate the framerate we've been asked to adapt to.
1075 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1076 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1077 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1078 mock_stats.input_frame_rate = fps;
1079 stats_proxy_->SetMockStats(mock_stats);
1080
1081 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1082 sink_.WaitForEncodedFrame(t);
1083 t += frame_interval_ms;
1084
mflodmancc3d4422017-08-03 08:27:51 -07001085 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001086 VerifyBalancedModeFpsRange(
1087 video_source_.sink_wants(),
1088 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1089 } while (video_source_.sink_wants().max_pixel_count <
1090 last_wants.max_pixel_count ||
1091 video_source_.sink_wants().max_framerate_fps <
1092 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001093
Jonathan Yubc771b72017-12-08 17:04:29 -08001094 // Verify that we've adapted all the way down.
1095 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001096 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001097 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1098 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001099 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001100 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1101 *video_source_.last_sent_height());
1102 EXPECT_EQ(kMinBalancedFramerateFps,
1103 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001104
Jonathan Yubc771b72017-12-08 17:04:29 -08001105 // Adapt back up the same number of times we adapted down.
1106 for (int i = 0; i < loop_count - 1; ++i) {
1107 last_wants = video_source_.sink_wants();
1108
1109 // Simulate the framerate we've been asked to adapt to.
1110 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1111 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1112 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1113 mock_stats.input_frame_rate = fps;
1114 stats_proxy_->SetMockStats(mock_stats);
1115
1116 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1117 sink_.WaitForEncodedFrame(t);
1118 t += frame_interval_ms;
1119
mflodmancc3d4422017-08-03 08:27:51 -07001120 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001121 VerifyBalancedModeFpsRange(
1122 video_source_.sink_wants(),
1123 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1124 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1125 last_wants.max_pixel_count ||
1126 video_source_.sink_wants().max_framerate_fps >
1127 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001128 }
1129
Åsa Persson8c1bf952018-09-13 10:42:19 +02001130 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001131 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001132 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001133 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1134 EXPECT_EQ((loop_count - 1) * 2,
1135 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001136
mflodmancc3d4422017-08-03 08:27:51 -07001137 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001138}
mflodmancc3d4422017-08-03 08:27:51 -07001139TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
1140 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001141 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001142
sprangc5d62e22017-04-02 23:53:04 -07001143 const int kFrameWidth = 1280;
1144 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001145
Åsa Persson8c1bf952018-09-13 10:42:19 +02001146 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001147
kthelgason5e13d412016-12-01 03:59:51 -08001148 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001149 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001150 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001151 frame_timestamp += kFrameIntervalMs;
1152
perkj803d97f2016-11-01 11:45:46 -07001153 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001154 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001155 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001156 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001157 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001158 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001159
asapersson0944a802017-04-07 00:57:58 -07001160 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001161 // wanted resolution.
1162 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1163 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1164 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001165 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001166
1167 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001168 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001169 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001170 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001171
sprangc5d62e22017-04-02 23:53:04 -07001172 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001173 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001174
sprangc5d62e22017-04-02 23:53:04 -07001175 // Force an input frame rate to be available, or the adaptation call won't
1176 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001177 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001178 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001179 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001180 stats_proxy_->SetMockStats(stats);
1181
mflodmancc3d4422017-08-03 08:27:51 -07001182 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001183 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001184 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001185 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001186 frame_timestamp += kFrameIntervalMs;
1187
1188 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001189 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001190 EXPECT_EQ(std::numeric_limits<int>::max(),
1191 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001192 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001193
asapersson02465b82017-04-10 01:12:52 -07001194 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001195 video_stream_encoder_->SetSource(&new_video_source,
1196 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001197 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001198
mflodmancc3d4422017-08-03 08:27:51 -07001199 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001200 new_video_source.IncomingCapturedFrame(
1201 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001202 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001203 frame_timestamp += kFrameIntervalMs;
1204
1205 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001206 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001207
1208 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001209 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001210 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001211 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1212 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001213 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001214 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001215
1216 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001217 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001218 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001219 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1220 EXPECT_EQ(std::numeric_limits<int>::max(),
1221 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001222 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001223
mflodmancc3d4422017-08-03 08:27:51 -07001224 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001225}
1226
mflodmancc3d4422017-08-03 08:27:51 -07001227TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
1228 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001229
asaperssonfab67072017-04-04 05:51:49 -07001230 const int kWidth = 1280;
1231 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001232 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001233 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001234 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1235 EXPECT_FALSE(stats.bw_limited_resolution);
1236 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1237
1238 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001239 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001240 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001241 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001242
1243 stats = stats_proxy_->GetStats();
1244 EXPECT_TRUE(stats.bw_limited_resolution);
1245 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1246
1247 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001248 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001249 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001250 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001251
1252 stats = stats_proxy_->GetStats();
1253 EXPECT_FALSE(stats.bw_limited_resolution);
1254 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1255 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1256
mflodmancc3d4422017-08-03 08:27:51 -07001257 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001258}
1259
mflodmancc3d4422017-08-03 08:27:51 -07001260TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
1261 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001262
1263 const int kWidth = 1280;
1264 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001265 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001266 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001267 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1268 EXPECT_FALSE(stats.cpu_limited_resolution);
1269 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1270
1271 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001272 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001273 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001274 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001275
1276 stats = stats_proxy_->GetStats();
1277 EXPECT_TRUE(stats.cpu_limited_resolution);
1278 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1279
1280 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001281 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001282 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001283 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001284
1285 stats = stats_proxy_->GetStats();
1286 EXPECT_FALSE(stats.cpu_limited_resolution);
1287 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001288 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001289
mflodmancc3d4422017-08-03 08:27:51 -07001290 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001291}
1292
mflodmancc3d4422017-08-03 08:27:51 -07001293TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
1294 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001295
asaperssonfab67072017-04-04 05:51:49 -07001296 const int kWidth = 1280;
1297 const int kHeight = 720;
1298 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001299 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001300 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001301 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001302 EXPECT_FALSE(stats.cpu_limited_resolution);
1303 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1304
asaperssonfab67072017-04-04 05:51:49 -07001305 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001306 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001307 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001308 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001309 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001310 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001311 EXPECT_TRUE(stats.cpu_limited_resolution);
1312 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1313
1314 // Set new source with adaptation still enabled.
1315 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001316 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001317 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001318
asaperssonfab67072017-04-04 05:51:49 -07001319 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001320 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001321 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001322 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001323 EXPECT_TRUE(stats.cpu_limited_resolution);
1324 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1325
1326 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001327 video_stream_encoder_->SetSource(&new_video_source,
1328 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001329
asaperssonfab67072017-04-04 05:51:49 -07001330 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001331 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001332 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001333 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001334 EXPECT_FALSE(stats.cpu_limited_resolution);
1335 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1336
1337 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001338 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001339 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001340
asaperssonfab67072017-04-04 05:51:49 -07001341 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001342 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001343 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001344 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001345 EXPECT_TRUE(stats.cpu_limited_resolution);
1346 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1347
asaperssonfab67072017-04-04 05:51:49 -07001348 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001349 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001350 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001351 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001352 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001353 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001354 EXPECT_FALSE(stats.cpu_limited_resolution);
1355 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001356 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001357
mflodmancc3d4422017-08-03 08:27:51 -07001358 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001359}
1360
mflodmancc3d4422017-08-03 08:27:51 -07001361TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
1362 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001363
asaperssonfab67072017-04-04 05:51:49 -07001364 const int kWidth = 1280;
1365 const int kHeight = 720;
1366 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001367 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001368 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001369 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001370 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001371 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001372
1373 // Set new source with adaptation still enabled.
1374 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001375 video_stream_encoder_->SetSource(&new_video_source,
1376 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001377
asaperssonfab67072017-04-04 05:51:49 -07001378 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001379 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001380 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001381 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001382 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001383 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001384
asaperssonfab67072017-04-04 05:51:49 -07001385 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001386 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001387 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001388 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001389 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001390 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001391 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001392 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001393
asaperssonfab67072017-04-04 05:51:49 -07001394 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001395 video_stream_encoder_->SetSource(&new_video_source,
1396 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001397
asaperssonfab67072017-04-04 05:51:49 -07001398 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001399 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001400 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001401 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001402 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001403 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001404
asapersson02465b82017-04-10 01:12:52 -07001405 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001406 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001407 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001408
asaperssonfab67072017-04-04 05:51:49 -07001409 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001410 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001411 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001412 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001413 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001414 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1415 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001416
mflodmancc3d4422017-08-03 08:27:51 -07001417 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001418}
1419
mflodmancc3d4422017-08-03 08:27:51 -07001420TEST_F(VideoStreamEncoderTest,
1421 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
1422 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001423
1424 const int kWidth = 1280;
1425 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001426 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001427 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001428 video_source_.IncomingCapturedFrame(
1429 CreateFrame(timestamp_ms, kWidth, kHeight));
1430 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001431 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1432 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1433 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1434
1435 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001436 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001437 timestamp_ms += kFrameIntervalMs;
1438 video_source_.IncomingCapturedFrame(
1439 CreateFrame(timestamp_ms, kWidth, kHeight));
1440 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001441 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1442 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1443 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1444
1445 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001446 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001447 timestamp_ms += kFrameIntervalMs;
1448 video_source_.IncomingCapturedFrame(
1449 CreateFrame(timestamp_ms, kWidth, kHeight));
1450 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001451 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1452 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1453 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1454
Niels Möller4db138e2018-04-19 09:04:13 +02001455 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001456 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001457
1458 VideoEncoderConfig video_encoder_config;
1459 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1460 // Make format different, to force recreation of encoder.
1461 video_encoder_config.video_format.parameters["foo"] = "foo";
1462 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001463 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001464 timestamp_ms += kFrameIntervalMs;
1465 video_source_.IncomingCapturedFrame(
1466 CreateFrame(timestamp_ms, kWidth, kHeight));
1467 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1470 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1471
mflodmancc3d4422017-08-03 08:27:51 -07001472 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07001473}
1474
mflodmancc3d4422017-08-03 08:27:51 -07001475TEST_F(VideoStreamEncoderTest,
1476 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
1477 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001478
asapersson0944a802017-04-07 00:57:58 -07001479 const int kWidth = 1280;
1480 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08001481 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07001482
asaperssonfab67072017-04-04 05:51:49 -07001483 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001484 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001485 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08001486 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001487 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08001488 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1489
asapersson02465b82017-04-10 01:12:52 -07001490 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001491 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001492 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001493 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08001494 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07001495 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001496 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001497 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1498
1499 // Set new source with adaptation still enabled.
1500 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001501 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001502 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001503
1504 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001505 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001506 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001507 stats = stats_proxy_->GetStats();
1508 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001509 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001510 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1511
sprangc5d62e22017-04-02 23:53:04 -07001512 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07001513 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001514 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001515 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001516 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001517 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001518 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07001519 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07001520 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07001521 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07001522 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1523
sprangc5d62e22017-04-02 23:53:04 -07001524 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07001525 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07001526 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1527 mock_stats.input_frame_rate = 30;
1528 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001529 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001530 stats_proxy_->ResetMockStats();
1531
1532 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001533 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001534 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001535
1536 // Framerate now adapted.
1537 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001538 EXPECT_FALSE(stats.cpu_limited_resolution);
1539 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001540 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1541
1542 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001543 video_stream_encoder_->SetSource(&new_video_source,
1544 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07001545 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001546 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001547 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001548
1549 stats = stats_proxy_->GetStats();
1550 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001551 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001552 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1553
1554 // Try to trigger overuse. Should not succeed.
1555 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07001556 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001557 stats_proxy_->ResetMockStats();
1558
1559 stats = stats_proxy_->GetStats();
1560 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001561 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001562 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
1563
1564 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001565 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001566 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07001567 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001568 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001569 stats = stats_proxy_->GetStats();
1570 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001571 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001572 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001573
1574 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001575 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001576 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001577 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07001578 stats = stats_proxy_->GetStats();
1579 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001580 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001581 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1582
1583 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001584 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001585 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001586 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001587 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001588 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001589 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07001590 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07001591 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001592 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001593 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
1594
1595 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07001596 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07001597 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07001598 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001599 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07001600 stats = stats_proxy_->GetStats();
1601 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001602 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07001603 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001604 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001605
mflodmancc3d4422017-08-03 08:27:51 -07001606 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001607}
1608
mflodmancc3d4422017-08-03 08:27:51 -07001609TEST_F(VideoStreamEncoderTest,
1610 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07001611 const int kWidth = 1280;
1612 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001613 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001614
asaperssonfab67072017-04-04 05:51:49 -07001615 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07001616 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08001617
asaperssonfab67072017-04-04 05:51:49 -07001618 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001619 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001620
asaperssonfab67072017-04-04 05:51:49 -07001621 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001622 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08001623
asaperssonfab67072017-04-04 05:51:49 -07001624 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001625 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08001626
kthelgason876222f2016-11-29 01:44:11 -08001627 // Expect a scale down.
1628 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07001629 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08001630
asapersson02465b82017-04-10 01:12:52 -07001631 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08001632 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001633 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001634 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001635
asaperssonfab67072017-04-04 05:51:49 -07001636 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07001637 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001638 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001639 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001640
asaperssonfab67072017-04-04 05:51:49 -07001641 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001642 EXPECT_EQ(std::numeric_limits<int>::max(),
1643 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001644
asaperssonfab67072017-04-04 05:51:49 -07001645 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07001646 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001647 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001648 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001649
asapersson02465b82017-04-10 01:12:52 -07001650 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07001651 EXPECT_EQ(std::numeric_limits<int>::max(),
1652 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08001653
mflodmancc3d4422017-08-03 08:27:51 -07001654 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001655}
1656
mflodmancc3d4422017-08-03 08:27:51 -07001657TEST_F(VideoStreamEncoderTest,
1658 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001659 const int kWidth = 1280;
1660 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001661 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001662
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001663 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001664 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001665 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001666 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001667
1668 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001669 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001670 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001671 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1672 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1673
1674 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001675 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07001676 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001677 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1678 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1679 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1680
1681 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001682 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07001683 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1684 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1685 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1686
mflodmancc3d4422017-08-03 08:27:51 -07001687 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001688}
1689
mflodmancc3d4422017-08-03 08:27:51 -07001690TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001691 const int kWidth = 1280;
1692 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001693 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001694
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001695 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001696 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001697 video_stream_encoder_->SetSource(&source,
1698 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001699 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1700 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001701 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001702
1703 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001704 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001705 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
1706 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1707 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1708 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
1709
1710 // Trigger adapt down for same input resolution, expect no change.
1711 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1712 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001713 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001714 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1715 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1716 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1717
1718 // Trigger adapt down for larger input resolution, expect no change.
1719 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
1720 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001721 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07001722 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
1723 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1724 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1725
mflodmancc3d4422017-08-03 08:27:51 -07001726 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001727}
1728
mflodmancc3d4422017-08-03 08:27:51 -07001729TEST_F(VideoStreamEncoderTest,
1730 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001731 const int kWidth = 1280;
1732 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001733 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001734
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001735 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001736 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001737 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001738 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001739
1740 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001741 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001742 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001743 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1744 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1745
1746 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001747 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001748 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1750 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1751
mflodmancc3d4422017-08-03 08:27:51 -07001752 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001753}
1754
mflodmancc3d4422017-08-03 08:27:51 -07001755TEST_F(VideoStreamEncoderTest,
1756 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07001757 const int kWidth = 1280;
1758 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001759 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001760
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001761 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001762 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07001763 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001764 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07001765
1766 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001767 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001768 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001769 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001770 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1771
1772 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001773 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001774 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001775 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07001776 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1777
mflodmancc3d4422017-08-03 08:27:51 -07001778 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001779}
1780
mflodmancc3d4422017-08-03 08:27:51 -07001781TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07001782 const int kWidth = 1280;
1783 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001784 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07001785
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001786 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07001787 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001788 video_stream_encoder_->SetSource(&source,
1789 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07001790
1791 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1792 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001793 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001794 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1795 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1796 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1797
1798 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001799 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001800 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001801 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1803 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1804
mflodmancc3d4422017-08-03 08:27:51 -07001805 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001806}
1807
mflodmancc3d4422017-08-03 08:27:51 -07001808TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07001809 const int kWidth = 1280;
1810 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001811 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001812
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001813 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07001814 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001815 video_stream_encoder_->SetSource(&source,
1816 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07001817
1818 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1819 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001820 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1822 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1823 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1824
1825 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07001826 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001827 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1829 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1831
mflodmancc3d4422017-08-03 08:27:51 -07001832 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001833}
1834
mflodmancc3d4422017-08-03 08:27:51 -07001835TEST_F(VideoStreamEncoderTest,
1836 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07001837 const int kWidth = 1280;
1838 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001839 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001840
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001841 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07001842 AdaptingFrameForwarder source;
1843 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001844 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001845 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07001846
1847 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001848 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001849 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001850 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1851 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1852
1853 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001854 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07001855 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001856 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07001857 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07001858 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1859 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1860
1861 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001862 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001863 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07001864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1865 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1866 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1867
mflodmancc3d4422017-08-03 08:27:51 -07001868 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07001869}
1870
mflodmancc3d4422017-08-03 08:27:51 -07001871TEST_F(VideoStreamEncoderTest,
1872 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07001873 const int kWidth = 1280;
1874 const int kHeight = 720;
1875 const int kInputFps = 30;
mflodmancc3d4422017-08-03 08:27:51 -07001876 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07001877
1878 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1879 stats.input_frame_rate = kInputFps;
1880 stats_proxy_->SetMockStats(stats);
1881
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001882 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07001883 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
1884 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001885 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001886
1887 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001888 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001889 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
1890 sink_.WaitForEncodedFrame(2);
1891 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
1892
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001893 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07001894 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001895 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001896 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001897 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001898
1899 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07001900 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07001901 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
1902 sink_.WaitForEncodedFrame(3);
1903 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
1904
1905 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001906 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001907 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07001908
mflodmancc3d4422017-08-03 08:27:51 -07001909 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07001910}
1911
mflodmancc3d4422017-08-03 08:27:51 -07001912TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07001913 const int kWidth = 1280;
1914 const int kHeight = 720;
1915 const size_t kNumFrames = 10;
1916
mflodmancc3d4422017-08-03 08:27:51 -07001917 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001918
asaperssond0de2952017-04-21 01:47:31 -07001919 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07001920 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07001921 video_source_.set_adaptation_enabled(true);
1922
1923 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1924 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
1925
1926 int downscales = 0;
1927 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02001928 video_source_.IncomingCapturedFrame(
1929 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
1930 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07001931
asaperssonfab67072017-04-04 05:51:49 -07001932 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07001933 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07001934 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07001935 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07001936
1937 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
1938 ++downscales;
1939
1940 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1941 EXPECT_EQ(downscales,
1942 stats_proxy_->GetStats().number_of_quality_adapt_changes);
1943 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08001944 }
mflodmancc3d4422017-08-03 08:27:51 -07001945 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07001946}
1947
mflodmancc3d4422017-08-03 08:27:51 -07001948TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07001949 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
1950 const int kWidth = 1280;
1951 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07001952 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07001953
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001954 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07001955 AdaptingFrameForwarder source;
1956 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07001957 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001958 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07001959
Åsa Persson8c1bf952018-09-13 10:42:19 +02001960 int64_t timestamp_ms = kFrameIntervalMs;
1961 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001962 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001963 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001964 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1965 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1966
1967 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001968 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001969 timestamp_ms += kFrameIntervalMs;
1970 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1971 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001972 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001973 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1974 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1975
1976 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001977 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001978 timestamp_ms += kFrameIntervalMs;
1979 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001980 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001981 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07001982 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1983 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1984
1985 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07001986 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001987 timestamp_ms += kFrameIntervalMs;
1988 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
1989 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07001990 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07001991 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1992 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1993
1994 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07001995 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001996 timestamp_ms += kFrameIntervalMs;
1997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07001998 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001999 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002000 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2001 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2002
mflodmancc3d4422017-08-03 08:27:51 -07002003 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002004}
2005
mflodmancc3d4422017-08-03 08:27:51 -07002006TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002007 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2008 const int kWidth = 1280;
2009 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002010 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002011
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002012 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002013 AdaptingFrameForwarder source;
2014 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002015 video_stream_encoder_->SetSource(&source,
2016 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002017
Åsa Persson8c1bf952018-09-13 10:42:19 +02002018 int64_t timestamp_ms = kFrameIntervalMs;
2019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002020 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002021 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2023 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2024
2025 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002026 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002027 timestamp_ms += kFrameIntervalMs;
2028 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2029 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002030 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2031 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2032 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2033
2034 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002035 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002036 timestamp_ms += kFrameIntervalMs;
2037 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002038 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002039 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002040 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2041 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2042
2043 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002044 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002045 timestamp_ms += kFrameIntervalMs;
2046 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2047 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002048 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2049 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2050 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2051
2052 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002053 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002054 timestamp_ms += kFrameIntervalMs;
2055 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002056 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002057 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002058 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2059 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2060
mflodmancc3d4422017-08-03 08:27:51 -07002061 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002062}
2063
mflodmancc3d4422017-08-03 08:27:51 -07002064TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002065 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2066 const int kWidth = 1280;
2067 const int kHeight = 720;
mflodmancc3d4422017-08-03 08:27:51 -07002068 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002069
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002070 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002071 AdaptingFrameForwarder source;
2072 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002073 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002074 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002075
Åsa Persson8c1bf952018-09-13 10:42:19 +02002076 int64_t timestamp_ms = kFrameIntervalMs;
2077 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002078 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002079 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002080 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2081 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2082 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2083 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2084
2085 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002086 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002087 timestamp_ms += kFrameIntervalMs;
2088 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2089 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002090 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002091 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2092 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2093 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2094 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2095
2096 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002097 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002098 timestamp_ms += kFrameIntervalMs;
2099 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2100 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002101 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002102 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2103 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2104 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2105 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2106
Jonathan Yubc771b72017-12-08 17:04:29 -08002107 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002108 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002109 timestamp_ms += kFrameIntervalMs;
2110 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2111 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002112 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002113 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2114 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002115 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002116 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2117
Jonathan Yubc771b72017-12-08 17:04:29 -08002118 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002119 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002120 timestamp_ms += kFrameIntervalMs;
2121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2122 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002123 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002124 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002125 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2126 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2127 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2128 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2129
Jonathan Yubc771b72017-12-08 17:04:29 -08002130 // Trigger quality adapt down, expect no change (min resolution reached).
2131 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002132 timestamp_ms += kFrameIntervalMs;
2133 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2134 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002135 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2136 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2137 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2138 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2139 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2140
2141 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002142 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002143 timestamp_ms += kFrameIntervalMs;
2144 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2145 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002146 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002147 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2148 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2149 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2150 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2151
2152 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2153 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002154 timestamp_ms += kFrameIntervalMs;
2155 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2156 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002157 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2158 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2159 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2160 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2161 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2162
2163 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2164 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002165 timestamp_ms += kFrameIntervalMs;
2166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2167 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002168 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002169 last_wants = source.sink_wants();
2170 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2171 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002172 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002173 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2174
2175 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002176 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002177 timestamp_ms += kFrameIntervalMs;
2178 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2179 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002180 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002181 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2182 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002183 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002184 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2185
2186 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002187 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002188 timestamp_ms += kFrameIntervalMs;
2189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002190 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002191 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002192 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002193 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002195 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002196 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002197
mflodmancc3d4422017-08-03 08:27:51 -07002198 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002199}
2200
mflodmancc3d4422017-08-03 08:27:51 -07002201TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002202 const int kWidth = 640;
2203 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002204
mflodmancc3d4422017-08-03 08:27:51 -07002205 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07002206
perkj803d97f2016-11-01 11:45:46 -07002207 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002208 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002209 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07002210 }
2211
mflodmancc3d4422017-08-03 08:27:51 -07002212 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002213 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002214 video_source_.IncomingCapturedFrame(CreateFrame(
2215 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002216 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07002217 }
2218
mflodmancc3d4422017-08-03 08:27:51 -07002219 video_stream_encoder_->Stop();
2220 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07002221 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08002222
perkj803d97f2016-11-01 11:45:46 -07002223 EXPECT_EQ(1,
2224 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2225 EXPECT_EQ(
2226 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
2227}
2228
mflodmancc3d4422017-08-03 08:27:51 -07002229TEST_F(VideoStreamEncoderTest,
2230 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
2231 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07002232 const int kWidth = 640;
2233 const int kHeight = 360;
2234
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002235 video_stream_encoder_->SetSource(&video_source_,
2236 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07002237
2238 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
2239 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002240 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07002241 }
2242
mflodmancc3d4422017-08-03 08:27:51 -07002243 video_stream_encoder_->Stop();
2244 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07002245 stats_proxy_.reset();
2246
2247 EXPECT_EQ(0,
2248 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
2249}
2250
mflodmancc3d4422017-08-03 08:27:51 -07002251TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07002252 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02002253 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08002254
2255 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02002256 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08002257 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
kthelgason2bc68642017-02-07 07:02:22 -08002258 .GetAllocation(kLowTargetBitrateBps, kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002259
sprang57c2fff2017-01-16 06:24:02 -08002260 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01002261 .Times(1);
mflodmancc3d4422017-08-03 08:27:51 -07002262 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08002263
sprang57c2fff2017-01-16 06:24:02 -08002264 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002265 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2266 WaitForEncodedFrame(rtc::TimeMillis());
2267 absl::optional<VideoBitrateAllocation> bitrate_allocation =
2268 fake_encoder_.GetAndResetLastBitrateAllocation();
2269 // Check that encoder has been updated too, not just allocation observer.
2270 EXPECT_EQ(bitrate_allocation->get_sum_bps(), kLowTargetBitrateBps);
2271 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002272
2273 // Not called on second frame.
2274 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2275 .Times(0);
2276 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01002277 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2278 WaitForEncodedFrame(rtc::TimeMillis());
2279 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08002280
2281 // Called after a process interval.
2282 const int64_t kProcessIntervalMs =
2283 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08002284 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
2285 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01002286 const int64_t start_time_ms = rtc::TimeMillis();
2287 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
2288 video_source_.IncomingCapturedFrame(
2289 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
2290 WaitForEncodedFrame(rtc::TimeMillis());
2291 fake_clock_.AdvanceTimeMicros(rtc::kNumMicrosecsPerMillisec / kDefaultFps);
2292 }
2293
2294 // Since rates are unchanged, encoder should not be reconfigured.
2295 EXPECT_FALSE(fake_encoder_.GetAndResetLastBitrateAllocation().has_value());
sprang57c2fff2017-01-16 06:24:02 -08002296
mflodmancc3d4422017-08-03 08:27:51 -07002297 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08002298}
2299
Niels Möller7dc26b72017-12-06 10:27:48 +01002300TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
2301 const int kFrameWidth = 1280;
2302 const int kFrameHeight = 720;
2303 const int kFramerate = 24;
2304
2305 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2306 test::FrameForwarder source;
2307 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002308 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002309
2310 // Insert a single frame, triggering initial configuration.
2311 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2312 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2313
2314 EXPECT_EQ(
2315 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2316 kDefaultFramerate);
2317
2318 // Trigger reconfigure encoder (without resetting the entire instance).
2319 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002320 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002321 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2322 video_encoder_config.number_of_streams = 1;
2323 video_encoder_config.video_stream_factory =
2324 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2325 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002326 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002327 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2328
2329 // Detector should be updated with fps limit from codec config.
2330 EXPECT_EQ(
2331 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2332 kFramerate);
2333
2334 // Trigger overuse, max framerate should be reduced.
2335 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2336 stats.input_frame_rate = kFramerate;
2337 stats_proxy_->SetMockStats(stats);
2338 video_stream_encoder_->TriggerCpuOveruse();
2339 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2340 int adapted_framerate =
2341 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2342 EXPECT_LT(adapted_framerate, kFramerate);
2343
2344 // Trigger underuse, max framerate should go back to codec configured fps.
2345 // Set extra low fps, to make sure it's actually reset, not just incremented.
2346 stats = stats_proxy_->GetStats();
2347 stats.input_frame_rate = adapted_framerate / 2;
2348 stats_proxy_->SetMockStats(stats);
2349 video_stream_encoder_->TriggerCpuNormalUsage();
2350 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2351 EXPECT_EQ(
2352 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2353 kFramerate);
2354
2355 video_stream_encoder_->Stop();
2356}
2357
2358TEST_F(VideoStreamEncoderTest,
2359 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
2360 const int kFrameWidth = 1280;
2361 const int kFrameHeight = 720;
2362 const int kLowFramerate = 15;
2363 const int kHighFramerate = 25;
2364
2365 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2366 test::FrameForwarder source;
2367 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002368 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01002369
2370 // Trigger initial configuration.
2371 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002372 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01002373 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2374 video_encoder_config.number_of_streams = 1;
2375 video_encoder_config.video_stream_factory =
2376 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
2377 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2378 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002379 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002380 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2381
2382 EXPECT_EQ(
2383 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2384 kLowFramerate);
2385
2386 // Trigger overuse, max framerate should be reduced.
2387 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2388 stats.input_frame_rate = kLowFramerate;
2389 stats_proxy_->SetMockStats(stats);
2390 video_stream_encoder_->TriggerCpuOveruse();
2391 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2392 int adapted_framerate =
2393 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
2394 EXPECT_LT(adapted_framerate, kLowFramerate);
2395
2396 // Reconfigure the encoder with a new (higher max framerate), max fps should
2397 // still respect the adaptation.
2398 video_encoder_config.video_stream_factory =
2399 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
2400 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
2401 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002402 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01002403 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2404
2405 EXPECT_EQ(
2406 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2407 adapted_framerate);
2408
2409 // Trigger underuse, max framerate should go back to codec configured fps.
2410 stats = stats_proxy_->GetStats();
2411 stats.input_frame_rate = adapted_framerate;
2412 stats_proxy_->SetMockStats(stats);
2413 video_stream_encoder_->TriggerCpuNormalUsage();
2414 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
2415 EXPECT_EQ(
2416 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2417 kHighFramerate);
2418
2419 video_stream_encoder_->Stop();
2420}
2421
mflodmancc3d4422017-08-03 08:27:51 -07002422TEST_F(VideoStreamEncoderTest,
2423 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07002424 const int kFrameWidth = 1280;
2425 const int kFrameHeight = 720;
2426 const int kFramerate = 24;
2427
mflodmancc3d4422017-08-03 08:27:51 -07002428 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07002429 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002430 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002431 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07002432
2433 // Trigger initial configuration.
2434 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02002435 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07002436 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
2437 video_encoder_config.number_of_streams = 1;
2438 video_encoder_config.video_stream_factory =
2439 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
2440 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07002441 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002442 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002443 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07002444
Niels Möller7dc26b72017-12-06 10:27:48 +01002445 EXPECT_EQ(
2446 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2447 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002448
2449 // Trigger overuse, max framerate should be reduced.
2450 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2451 stats.input_frame_rate = kFramerate;
2452 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07002453 video_stream_encoder_->TriggerCpuOveruse();
2454 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002455 int adapted_framerate =
2456 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07002457 EXPECT_LT(adapted_framerate, kFramerate);
2458
2459 // Change degradation preference to not enable framerate scaling. Target
2460 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07002461 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002462 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07002463 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01002464 EXPECT_EQ(
2465 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
2466 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07002467
mflodmancc3d4422017-08-03 08:27:51 -07002468 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07002469}
2470
mflodmancc3d4422017-08-03 08:27:51 -07002471TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002472 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002473 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002474 const int kWidth = 640;
2475 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002476
asaperssonfab67072017-04-04 05:51:49 -07002477 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002478
2479 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002480 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002481
2482 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07002483 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002484
sprangc5d62e22017-04-02 23:53:04 -07002485 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08002486
asaperssonfab67072017-04-04 05:51:49 -07002487 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08002488 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002489 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08002490
2491 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07002492 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002493
sprangc5d62e22017-04-02 23:53:04 -07002494 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08002495
mflodmancc3d4422017-08-03 08:27:51 -07002496 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002497}
2498
mflodmancc3d4422017-08-03 08:27:51 -07002499TEST_F(VideoStreamEncoderTest,
2500 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07002501 const int kTooLowBitrateForFrameSizeBps = 10000;
mflodmancc3d4422017-08-03 08:27:51 -07002502 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002503 const int kWidth = 640;
2504 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08002505
2506 // We expect the n initial frames to get dropped.
2507 int i;
2508 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07002509 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002510 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08002511 }
2512 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07002513 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002514 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08002515
2516 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07002517 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08002518
mflodmancc3d4422017-08-03 08:27:51 -07002519 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002520}
2521
mflodmancc3d4422017-08-03 08:27:51 -07002522TEST_F(VideoStreamEncoderTest,
2523 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07002524 const int kWidth = 640;
2525 const int kHeight = 360;
mflodmancc3d4422017-08-03 08:27:51 -07002526 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08002527
2528 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07002529 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002530 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08002531
asaperssonfab67072017-04-04 05:51:49 -07002532 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08002533 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002534 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08002535
mflodmancc3d4422017-08-03 08:27:51 -07002536 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08002537}
2538
mflodmancc3d4422017-08-03 08:27:51 -07002539TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07002540 const int kWidth = 640;
2541 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08002542 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002543
2544 VideoEncoderConfig video_encoder_config;
2545 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2546 // Make format different, to force recreation of encoder.
2547 video_encoder_config.video_format.parameters["foo"] = "foo";
2548 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002549 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07002550 video_stream_encoder_->OnBitrateUpdated(kLowTargetBitrateBps, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002551
kthelgasonb83797b2017-02-14 11:57:25 -08002552 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 video_stream_encoder_->SetSource(&video_source_,
2554 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08002555
asaperssonfab67072017-04-04 05:51:49 -07002556 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08002557 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07002558 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08002559
mflodmancc3d4422017-08-03 08:27:51 -07002560 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08002561 fake_encoder_.SetQualityScaling(true);
2562}
2563
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02002564TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
2565 webrtc::test::ScopedFieldTrials field_trials(
2566 "WebRTC-InitialFramedrop/Enabled/");
2567 // Reset encoder for field trials to take effect.
2568 ConfigureEncoder(video_encoder_config_.Copy());
2569 const int kTooLowBitrateForFrameSizeBps = 10000;
2570 const int kWidth = 640;
2571 const int kHeight = 360;
2572
2573 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2574 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2575 // Frame should not be dropped.
2576 WaitForEncodedFrame(1);
2577
2578 video_stream_encoder_->OnBitrateUpdated(kTooLowBitrateForFrameSizeBps, 0, 0);
2579 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2580 // Expect to drop this frame, the wait should time out.
2581 ExpectDroppedFrame();
2582
2583 // Expect the sink_wants to specify a scaled frame.
2584 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
2585 video_stream_encoder_->Stop();
2586}
2587
mflodmancc3d4422017-08-03 08:27:51 -07002588TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002589 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
2590 const int kTooSmallWidth = 10;
2591 const int kTooSmallHeight = 10;
mflodmancc3d4422017-08-03 08:27:51 -07002592 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002593
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002594 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002595 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002596 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002597 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002598 VerifyNoLimitation(source.sink_wants());
2599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2600
2601 // Trigger adapt down, too small frame, expect no change.
2602 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002603 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002604 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002605 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002606 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2607 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2608
mflodmancc3d4422017-08-03 08:27:51 -07002609 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002610}
2611
mflodmancc3d4422017-08-03 08:27:51 -07002612TEST_F(VideoStreamEncoderTest,
2613 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002614 const int kTooSmallWidth = 10;
2615 const int kTooSmallHeight = 10;
2616 const int kFpsLimit = 7;
mflodmancc3d4422017-08-03 08:27:51 -07002617 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002618
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002619 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002620 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002621 video_stream_encoder_->SetSource(&source,
2622 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002623 VerifyNoLimitation(source.sink_wants());
2624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2625 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2626
2627 // Trigger adapt down, expect limited framerate.
2628 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002629 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07002630 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002631 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2632 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2633 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2634 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2635
2636 // Trigger adapt down, too small frame, expect no change.
2637 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07002638 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002639 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002640 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
2641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2643 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2644
mflodmancc3d4422017-08-03 08:27:51 -07002645 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002646}
2647
mflodmancc3d4422017-08-03 08:27:51 -07002648TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07002649 fake_encoder_.ForceInitEncodeFailure(true);
mflodmancc3d4422017-08-03 08:27:51 -07002650 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02002651 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07002652 const int kFrameWidth = 1280;
2653 const int kFrameHeight = 720;
2654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002656 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07002657 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002658}
2659
sprangb1ca0732017-02-01 08:38:12 -08002660// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07002661TEST_F(VideoStreamEncoderTest,
2662 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
2663 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08002664
2665 const int kFrameWidth = 1280;
2666 const int kFrameHeight = 720;
2667 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07002668 // requested by
2669 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08002670 video_source_.set_adaptation_enabled(true);
2671
2672 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002673 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002674 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002675
2676 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07002677 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08002678 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002679 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002680 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08002681
asaperssonfab67072017-04-04 05:51:49 -07002682 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002683 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08002684 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02002685 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002686 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08002687
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08002689}
sprangfe627f32017-03-29 08:24:59 -07002690
mflodmancc3d4422017-08-03 08:27:51 -07002691TEST_F(VideoStreamEncoderTest,
2692 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07002693 const int kFrameWidth = 1280;
2694 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002695
mflodmancc3d4422017-08-03 08:27:51 -07002696 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2697 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002698 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002699 video_source_.set_adaptation_enabled(true);
2700
sprang4847ae62017-06-27 07:06:52 -07002701 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002702
2703 video_source_.IncomingCapturedFrame(
2704 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002705 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002706
2707 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002709
2710 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07002711 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002712 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002713 video_source_.IncomingCapturedFrame(
2714 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002715 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07002716 }
2717
2718 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07002719 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002720 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002721 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002722 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002723 video_source_.IncomingCapturedFrame(
2724 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002725 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002726 ++num_frames_dropped;
2727 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002728 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002729 }
2730 }
2731
sprang4847ae62017-06-27 07:06:52 -07002732 // Add some slack to account for frames dropped by the frame dropper.
2733 const int kErrorMargin = 1;
2734 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002735 kErrorMargin);
2736
2737 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07002738 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002739 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002740 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002741 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002742 video_source_.IncomingCapturedFrame(
2743 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002744 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002745 ++num_frames_dropped;
2746 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002747 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002748 }
2749 }
sprang4847ae62017-06-27 07:06:52 -07002750 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07002751 kErrorMargin);
2752
2753 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07002754 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002755 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002756 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002757 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002758 video_source_.IncomingCapturedFrame(
2759 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002760 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002761 ++num_frames_dropped;
2762 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002763 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002764 }
2765 }
sprang4847ae62017-06-27 07:06:52 -07002766 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07002767 kErrorMargin);
2768
2769 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07002770 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002771 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07002772 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07002773 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002774 video_source_.IncomingCapturedFrame(
2775 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002776 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07002777 ++num_frames_dropped;
2778 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01002779 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07002780 }
2781 }
2782 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
2783
mflodmancc3d4422017-08-03 08:27:51 -07002784 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002785}
2786
mflodmancc3d4422017-08-03 08:27:51 -07002787TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07002788 const int kFramerateFps = 5;
2789 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07002790 const int kFrameWidth = 1280;
2791 const int kFrameHeight = 720;
2792
sprang4847ae62017-06-27 07:06:52 -07002793 // Reconfigure encoder with two temporal layers and screensharing, which will
2794 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02002795 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07002796
mflodmancc3d4422017-08-03 08:27:51 -07002797 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
2798 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002799 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002800 video_source_.set_adaptation_enabled(true);
2801
sprang4847ae62017-06-27 07:06:52 -07002802 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07002803
2804 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08002805 rtc::VideoSinkWants last_wants;
2806 do {
2807 last_wants = video_source_.sink_wants();
2808
sprangc5d62e22017-04-02 23:53:04 -07002809 // Insert frames to get a new fps estimate...
2810 for (int j = 0; j < kFramerateFps; ++j) {
2811 video_source_.IncomingCapturedFrame(
2812 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08002813 if (video_source_.last_sent_width()) {
2814 sink_.WaitForEncodedFrame(timestamp_ms);
2815 }
sprangc5d62e22017-04-02 23:53:04 -07002816 timestamp_ms += kFrameIntervalMs;
Yves Gerey665174f2018-06-19 15:03:05 +02002817 fake_clock_.AdvanceTimeMicros(kFrameIntervalMs *
2818 rtc::kNumMicrosecsPerMillisec);
sprangc5d62e22017-04-02 23:53:04 -07002819 }
2820 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07002821 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08002822 } while (video_source_.sink_wants().max_framerate_fps <
2823 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002824
Jonathan Yubc771b72017-12-08 17:04:29 -08002825 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07002826
mflodmancc3d4422017-08-03 08:27:51 -07002827 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07002828}
asaperssonf7e294d2017-06-13 23:25:22 -07002829
mflodmancc3d4422017-08-03 08:27:51 -07002830TEST_F(VideoStreamEncoderTest,
2831 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002832 const int kWidth = 1280;
2833 const int kHeight = 720;
2834 const int64_t kFrameIntervalMs = 150;
2835 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07002836 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002837
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002838 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002839 AdaptingFrameForwarder source;
2840 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002841 video_stream_encoder_->SetSource(&source,
2842 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002843 timestamp_ms += kFrameIntervalMs;
2844 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002845 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002846 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002847 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2849 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2850
2851 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002852 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002853 timestamp_ms += kFrameIntervalMs;
2854 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002855 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002856 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2857 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2858 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2859 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2860
2861 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002862 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002863 timestamp_ms += kFrameIntervalMs;
2864 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002865 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002866 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
2867 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2868 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2869 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2870
2871 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002872 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002873 timestamp_ms += kFrameIntervalMs;
2874 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002875 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002876 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2879 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2880
2881 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002882 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002883 timestamp_ms += kFrameIntervalMs;
2884 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002885 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002886 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2887 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2888 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2889 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2890
2891 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002892 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002893 timestamp_ms += kFrameIntervalMs;
2894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002895 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002896 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2897 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2898 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2899 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2900
2901 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002902 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002903 timestamp_ms += kFrameIntervalMs;
2904 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002905 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002906 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2907 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2908 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2909 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2910
2911 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07002912 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002913 timestamp_ms += kFrameIntervalMs;
2914 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002915 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002916 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2917 rtc::VideoSinkWants last_wants = source.sink_wants();
2918 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2919 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2920 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2921
2922 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002924 timestamp_ms += kFrameIntervalMs;
2925 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002926 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002927 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
2928 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2929 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2930 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2931
2932 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002933 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002934 timestamp_ms += kFrameIntervalMs;
2935 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002936 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002937 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2938 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2939 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2940 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2941
2942 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07002943 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002944 timestamp_ms += kFrameIntervalMs;
2945 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002946 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002947 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2948 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2949 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2950 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2951
2952 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002953 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002954 timestamp_ms += kFrameIntervalMs;
2955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002956 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002957 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2958 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2959 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2960 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2961
2962 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07002963 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002964 timestamp_ms += kFrameIntervalMs;
2965 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002966 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002967 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2968 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2969 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
2970 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2971
2972 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002973 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002974 timestamp_ms += kFrameIntervalMs;
2975 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002976 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002977 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
2978 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2979 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2980 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2981
2982 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002983 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002984 timestamp_ms += kFrameIntervalMs;
2985 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002986 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002987 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2988 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2989 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
2990 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2991
2992 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07002993 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07002994 timestamp_ms += kFrameIntervalMs;
2995 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002996 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07002997 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002998 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002999 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3000 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3001 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3002
3003 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003004 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003005 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003006 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3007
mflodmancc3d4422017-08-03 08:27:51 -07003008 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003009}
3010
mflodmancc3d4422017-08-03 08:27:51 -07003011TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003012 const int kWidth = 1280;
3013 const int kHeight = 720;
3014 const int64_t kFrameIntervalMs = 150;
3015 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003016 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003017
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003018 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003019 AdaptingFrameForwarder source;
3020 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003021 video_stream_encoder_->SetSource(&source,
3022 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003023 timestamp_ms += kFrameIntervalMs;
3024 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003025 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003026 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003027 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3028 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3029 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3030 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3031 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3032 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3033
3034 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003035 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003036 timestamp_ms += kFrameIntervalMs;
3037 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003038 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003039 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3040 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3042 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3043 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3044 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3045 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3046
3047 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003048 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003049 timestamp_ms += kFrameIntervalMs;
3050 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003051 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003052 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3053 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3054 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3055 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3056 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3057 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3058 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3059
3060 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003061 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003062 timestamp_ms += kFrameIntervalMs;
3063 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003064 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003065 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3066 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3067 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3068 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3069 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3070 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3071 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3072
3073 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003074 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003075 timestamp_ms += kFrameIntervalMs;
3076 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003077 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003078 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3079 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3080 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3081 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3082 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3083 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3084 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3085
3086 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003087 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003088 timestamp_ms += kFrameIntervalMs;
3089 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003090 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003091 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3092 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3093 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3094 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3095 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3096 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3097 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3098
3099 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003100 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003101 timestamp_ms += kFrameIntervalMs;
3102 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003103 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003104 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003105 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003106 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3107 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3108 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3109 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3110 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3111 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3112
3113 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003114 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003115 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003116 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3117 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3118
mflodmancc3d4422017-08-03 08:27:51 -07003119 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003120}
3121
mflodmancc3d4422017-08-03 08:27:51 -07003122TEST_F(VideoStreamEncoderTest,
3123 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07003124 const int kWidth = 640;
3125 const int kHeight = 360;
3126 const int kFpsLimit = 15;
3127 const int64_t kFrameIntervalMs = 150;
3128 int64_t timestamp_ms = kFrameIntervalMs;
mflodmancc3d4422017-08-03 08:27:51 -07003129 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003130
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003131 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003132 AdaptingFrameForwarder source;
3133 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003134 video_stream_encoder_->SetSource(&source,
3135 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003136 timestamp_ms += kFrameIntervalMs;
3137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003138 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003139 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003140 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3141 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3142 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3143 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3144 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3145 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3146
3147 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003148 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003149 timestamp_ms += kFrameIntervalMs;
3150 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003151 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003152 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3153 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3154 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3155 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3156 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3157 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3158 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3159
3160 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003161 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003162 timestamp_ms += kFrameIntervalMs;
3163 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003164 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003165 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3166 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3167 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3168 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3169 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3170 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3171 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3172
3173 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003174 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003175 timestamp_ms += kFrameIntervalMs;
3176 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003177 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003178 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3179 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3180 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3181 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3182 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3183 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3184 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3185
3186 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003188 timestamp_ms += kFrameIntervalMs;
3189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003190 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003191 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003192 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3194 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3195 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3196 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3197 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3198
3199 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003200 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003201 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003202 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3203 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3204
mflodmancc3d4422017-08-03 08:27:51 -07003205 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003206}
3207
mflodmancc3d4422017-08-03 08:27:51 -07003208TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07003209 // Simulates simulcast behavior and makes highest stream resolutions divisible
3210 // by 4.
3211 class CroppingVideoStreamFactory
3212 : public VideoEncoderConfig::VideoStreamFactoryInterface {
3213 public:
3214 explicit CroppingVideoStreamFactory(size_t num_temporal_layers,
3215 int framerate)
3216 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
3217 EXPECT_GT(num_temporal_layers, 0u);
3218 EXPECT_GT(framerate, 0);
3219 }
3220
3221 private:
3222 std::vector<VideoStream> CreateEncoderStreams(
3223 int width,
3224 int height,
3225 const VideoEncoderConfig& encoder_config) override {
Yves Gerey665174f2018-06-19 15:03:05 +02003226 std::vector<VideoStream> streams = test::CreateVideoStreams(
3227 width - width % 4, height - height % 4, encoder_config);
ilnik6b826ef2017-06-16 06:53:48 -07003228 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +01003229 stream.num_temporal_layers = num_temporal_layers_;
ilnik6b826ef2017-06-16 06:53:48 -07003230 stream.max_framerate = framerate_;
3231 }
3232 return streams;
3233 }
3234
3235 const size_t num_temporal_layers_;
3236 const int framerate_;
3237 };
3238
3239 const int kFrameWidth = 1920;
3240 const int kFrameHeight = 1080;
3241 // 3/4 of 1920.
3242 const int kAdaptedFrameWidth = 1440;
3243 // 3/4 of 1080 rounded down to multiple of 4.
3244 const int kAdaptedFrameHeight = 808;
3245 const int kFramerate = 24;
3246
mflodmancc3d4422017-08-03 08:27:51 -07003247 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07003248 // Trigger reconfigure encoder (without resetting the entire instance).
3249 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003250 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07003251 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3252 video_encoder_config.number_of_streams = 1;
3253 video_encoder_config.video_stream_factory =
3254 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003256 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003257 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07003258
3259 video_source_.set_adaptation_enabled(true);
3260
3261 video_source_.IncomingCapturedFrame(
3262 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003263 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003264
3265 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003266 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07003267 video_source_.IncomingCapturedFrame(
3268 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003269 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07003270
mflodmancc3d4422017-08-03 08:27:51 -07003271 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07003272}
3273
mflodmancc3d4422017-08-03 08:27:51 -07003274TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07003275 const int kFrameWidth = 1280;
3276 const int kFrameHeight = 720;
3277 const int kLowFps = 2;
3278 const int kHighFps = 30;
3279
mflodmancc3d4422017-08-03 08:27:51 -07003280 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003281
3282 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3283 max_framerate_ = kLowFps;
3284
3285 // Insert 2 seconds of 2fps video.
3286 for (int i = 0; i < kLowFps * 2; ++i) {
3287 video_source_.IncomingCapturedFrame(
3288 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3289 WaitForEncodedFrame(timestamp_ms);
3290 timestamp_ms += 1000 / kLowFps;
3291 }
3292
3293 // Make sure encoder is updated with new target.
mflodmancc3d4422017-08-03 08:27:51 -07003294 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003295 video_source_.IncomingCapturedFrame(
3296 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3297 WaitForEncodedFrame(timestamp_ms);
3298 timestamp_ms += 1000 / kLowFps;
3299
3300 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
3301
3302 // Insert 30fps frames for just a little more than the forced update period.
3303 const int kVcmTimerIntervalFrames =
3304 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
3305 const int kFrameIntervalMs = 1000 / kHighFps;
3306 max_framerate_ = kHighFps;
3307 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
3308 video_source_.IncomingCapturedFrame(
3309 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3310 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
3311 // be dropped if the encoder hans't been updated with the new higher target
3312 // framerate yet, causing it to overshoot the target bitrate and then
3313 // suffering the wrath of the media optimizer.
3314 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
3315 timestamp_ms += kFrameIntervalMs;
3316 }
3317
3318 // Don expect correct measurement just yet, but it should be higher than
3319 // before.
3320 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
3321
mflodmancc3d4422017-08-03 08:27:51 -07003322 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003323}
3324
mflodmancc3d4422017-08-03 08:27:51 -07003325TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07003326 const int kFrameWidth = 1280;
3327 const int kFrameHeight = 720;
3328 const int kTargetBitrateBps = 1000000;
3329
3330 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003331 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
mflodmancc3d4422017-08-03 08:27:51 -07003332 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3333 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07003334
3335 // Insert a first video frame, causes another bitrate update.
3336 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3337 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
3338 video_source_.IncomingCapturedFrame(
3339 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3340 WaitForEncodedFrame(timestamp_ms);
3341
3342 // Next, simulate video suspension due to pacer queue overrun.
mflodmancc3d4422017-08-03 08:27:51 -07003343 video_stream_encoder_->OnBitrateUpdated(0, 0, 1);
sprang4847ae62017-06-27 07:06:52 -07003344
3345 // Skip ahead until a new periodic parameter update should have occured.
3346 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
3347 fake_clock_.AdvanceTimeMicros(
3348 vcm::VCMProcessTimer::kDefaultProcessIntervalMs *
3349 rtc::kNumMicrosecsPerMillisec);
3350
3351 // Bitrate observer should not be called.
3352 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
3353 video_source_.IncomingCapturedFrame(
3354 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3355 ExpectDroppedFrame();
3356
mflodmancc3d4422017-08-03 08:27:51 -07003357 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07003358}
ilnik6b826ef2017-06-16 06:53:48 -07003359
Niels Möller4db138e2018-04-19 09:04:13 +02003360TEST_F(VideoStreamEncoderTest,
3361 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
3362 const int kFrameWidth = 1280;
3363 const int kFrameHeight = 720;
3364 const CpuOveruseOptions default_options;
3365 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3366 video_source_.IncomingCapturedFrame(
3367 CreateFrame(1, kFrameWidth, kFrameHeight));
3368 WaitForEncodedFrame(1);
3369 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3370 .low_encode_usage_threshold_percent,
3371 default_options.low_encode_usage_threshold_percent);
3372 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3373 .high_encode_usage_threshold_percent,
3374 default_options.high_encode_usage_threshold_percent);
3375 video_stream_encoder_->Stop();
3376}
3377
3378TEST_F(VideoStreamEncoderTest,
3379 HigherCpuAdaptationThresholdsForHardwareEncoder) {
3380 const int kFrameWidth = 1280;
3381 const int kFrameHeight = 720;
3382 CpuOveruseOptions hardware_options;
3383 hardware_options.low_encode_usage_threshold_percent = 150;
3384 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01003385 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02003386
3387 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3388 video_source_.IncomingCapturedFrame(
3389 CreateFrame(1, kFrameWidth, kFrameHeight));
3390 WaitForEncodedFrame(1);
3391 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3392 .low_encode_usage_threshold_percent,
3393 hardware_options.low_encode_usage_threshold_percent);
3394 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
3395 .high_encode_usage_threshold_percent,
3396 hardware_options.high_encode_usage_threshold_percent);
3397 video_stream_encoder_->Stop();
3398}
3399
Niels Möller6bb5ab92019-01-11 11:11:10 +01003400TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
3401 const int kFrameWidth = 320;
3402 const int kFrameHeight = 240;
3403 const int kFps = 30;
3404 const int kTargetBitrateBps = 120000;
3405 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
3406
3407 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3408
3409 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3410 max_framerate_ = kFps;
3411
3412 // Insert 3 seconds of video, verify number of drops with normal bitrate.
3413 fake_encoder_.SimulateOvershoot(1.0);
3414 int num_dropped = 0;
3415 for (int i = 0; i < kNumFramesInRun; ++i) {
3416 video_source_.IncomingCapturedFrame(
3417 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3418 // Wait up to two frame durations for a frame to arrive.
3419 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3420 ++num_dropped;
3421 }
3422 timestamp_ms += 1000 / kFps;
3423 }
3424
Erik Språnga8d48ab2019-02-08 14:17:40 +01003425 // Framerate should be measured to be near the expected target rate.
3426 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3427
3428 // Frame drops should be within 5% of expected 0%.
3429 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003430
3431 // Make encoder produce frames at double the expected bitrate during 3 seconds
3432 // of video, verify number of drops. Rate needs to be slightly changed in
3433 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01003434 double overshoot_factor = 2.0;
3435 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
3436 // With bitrate adjuster, when need to overshoot even more to trigger
3437 // frame dropping.
3438 overshoot_factor *= 2;
3439 }
3440 fake_encoder_.SimulateOvershoot(overshoot_factor);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003441 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps + 1000, 0, 0);
3442 num_dropped = 0;
3443 for (int i = 0; i < kNumFramesInRun; ++i) {
3444 video_source_.IncomingCapturedFrame(
3445 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3446 // Wait up to two frame durations for a frame to arrive.
3447 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
3448 ++num_dropped;
3449 }
3450 timestamp_ms += 1000 / kFps;
3451 }
3452
Erik Språnga8d48ab2019-02-08 14:17:40 +01003453 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3454
3455 // Target framerate should be still be near the expected target, despite
3456 // the frame drops.
3457 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
3458
3459 // Frame drops should be within 5% of expected 50%.
3460 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01003461
3462 video_stream_encoder_->Stop();
3463}
3464
3465TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
3466 const int kFrameWidth = 320;
3467 const int kFrameHeight = 240;
3468 const int kActualInputFps = 24;
3469 const int kTargetBitrateBps = 120000;
3470
3471 ASSERT_GT(max_framerate_, kActualInputFps);
3472
3473 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
3474 max_framerate_ = kActualInputFps;
3475 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3476
3477 // Insert 3 seconds of video, with an input fps lower than configured max.
3478 for (int i = 0; i < kActualInputFps * 3; ++i) {
3479 video_source_.IncomingCapturedFrame(
3480 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
3481 // Wait up to two frame durations for a frame to arrive.
3482 WaitForEncodedFrame(timestamp_ms);
3483 timestamp_ms += 1000 / kActualInputFps;
3484 }
3485
3486 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
3487
3488 video_stream_encoder_->Stop();
3489}
3490
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01003491TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
3492 VideoFrame::UpdateRect rect;
3493 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3494
3495 fake_encoder_.BlockNextEncode();
3496 video_source_.IncomingCapturedFrame(
3497 CreateFrameWithUpdatedPixel(1, nullptr, 0));
3498 WaitForEncodedFrame(1);
3499 // On the very first frame full update should be forced.
3500 rect = fake_encoder_.GetLastUpdateRect();
3501 EXPECT_EQ(rect.offset_x, 0);
3502 EXPECT_EQ(rect.offset_y, 0);
3503 EXPECT_EQ(rect.height, codec_height_);
3504 EXPECT_EQ(rect.width, codec_width_);
3505 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
3506 // call to ContinueEncode.
3507 video_source_.IncomingCapturedFrame(
3508 CreateFrameWithUpdatedPixel(2, nullptr, 1));
3509 ExpectDroppedFrame();
3510 video_source_.IncomingCapturedFrame(
3511 CreateFrameWithUpdatedPixel(3, nullptr, 10));
3512 ExpectDroppedFrame();
3513 fake_encoder_.ContinueEncode();
3514 WaitForEncodedFrame(3);
3515 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
3516 rect = fake_encoder_.GetLastUpdateRect();
3517 EXPECT_EQ(rect.offset_x, 1);
3518 EXPECT_EQ(rect.offset_y, 0);
3519 EXPECT_EQ(rect.width, 10);
3520 EXPECT_EQ(rect.height, 1);
3521
3522 video_source_.IncomingCapturedFrame(
3523 CreateFrameWithUpdatedPixel(4, nullptr, 0));
3524 WaitForEncodedFrame(4);
3525 // Previous frame was encoded, so no accumulation should happen.
3526 rect = fake_encoder_.GetLastUpdateRect();
3527 EXPECT_EQ(rect.offset_x, 0);
3528 EXPECT_EQ(rect.offset_y, 0);
3529 EXPECT_EQ(rect.width, 1);
3530 EXPECT_EQ(rect.height, 1);
3531
3532 video_stream_encoder_->Stop();
3533}
3534
Erik Språngd7329ca2019-02-21 21:19:53 +01003535TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
3536 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3537
3538 // First frame is always keyframe.
3539 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3540 WaitForEncodedFrame(1);
3541 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3542 testing::ElementsAre(FrameType{kVideoFrameKey}));
3543
3544 // Insert delta frame.
3545 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3546 WaitForEncodedFrame(2);
3547 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3548 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3549
3550 // Request next frame be a key-frame.
3551 video_stream_encoder_->SendKeyFrame();
3552 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3553 WaitForEncodedFrame(3);
3554 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3555 testing::ElementsAre(FrameType{kVideoFrameKey}));
3556
3557 video_stream_encoder_->Stop();
3558}
3559
3560TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
3561 // Setup simulcast with three streams.
3562 ResetEncoder("VP8", 3, 1, 1, false);
3563 video_stream_encoder_->OnBitrateUpdated(kSimulcastTargetBitrateBps, 0, 0);
3564 // Wait for all three layers before triggering event.
3565 sink_.SetNumExpectedLayers(3);
3566
3567 // First frame is always keyframe.
3568 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
3569 WaitForEncodedFrame(1);
3570 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3571 testing::ElementsAreArray(
3572 {kVideoFrameKey, kVideoFrameKey, kVideoFrameKey}));
3573
3574 // Insert delta frame.
3575 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
3576 WaitForEncodedFrame(2);
3577 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3578 testing::ElementsAreArray(
3579 {kVideoFrameDelta, kVideoFrameDelta, kVideoFrameDelta}));
3580
3581 // Request next frame be a key-frame.
3582 // Only first stream is configured to produce key-frame.
3583 video_stream_encoder_->SendKeyFrame();
3584 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
3585 WaitForEncodedFrame(3);
3586 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3587 testing::ElementsAreArray(
3588 {kVideoFrameKey, kVideoFrameDelta, kVideoFrameDelta}));
3589
3590 video_stream_encoder_->Stop();
3591}
3592
3593TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
3594 // Configure internal source factory and setup test again.
3595 encoder_factory_.SetHasInternalSource(true);
3596 ResetEncoder("VP8", 1, 1, 1, false);
3597 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3598
3599 // Call encoder directly, simulating internal source where encoded frame
3600 // callback in VideoStreamEncoder is called despite no OnFrame().
3601 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
3602 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3603 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3604 testing::ElementsAre(FrameType{kVideoFrameKey}));
3605
3606 const std::vector<FrameType> kDeltaFrame = {kVideoFrameDelta};
3607 // Need to set timestamp manually since manually for injected frame.
3608 VideoFrame frame = CreateFrame(101, nullptr);
3609 frame.set_timestamp(101);
3610 fake_encoder_.InjectFrame(frame, false);
3611 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3612 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3613 testing::ElementsAre(FrameType{kVideoFrameDelta}));
3614
3615 // Request key-frame. The forces a dummy frame down into the encoder.
3616 fake_encoder_.ExpectNullFrame();
3617 video_stream_encoder_->SendKeyFrame();
3618 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
3619 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
3620 testing::ElementsAre(FrameType{kVideoFrameKey}));
3621
3622 video_stream_encoder_->Stop();
3623}
Erik Språngb7cb7b52019-02-26 15:52:33 +01003624
3625TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
3626 // Configure internal source factory and setup test again.
3627 encoder_factory_.SetHasInternalSource(true);
3628 ResetEncoder("VP8", 1, 1, 1, false);
3629 video_stream_encoder_->OnBitrateUpdated(kTargetBitrateBps, 0, 0);
3630
3631 int64_t timestamp = 1;
3632 EncodedImage image;
3633 image.Allocate(kTargetBitrateBps / kDefaultFramerate / 8);
3634 image.capture_time_ms_ = ++timestamp;
3635 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
3636 const int64_t kEncodeFinishDelayMs = 10;
3637 image.timing_.encode_start_ms = timestamp;
3638 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
3639 fake_encoder_.InjectEncodedImage(image);
3640 // Wait for frame without incrementing clock.
3641 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
3642 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
3643 // capture timestamp should be kEncodeFinishDelayMs in the past.
3644 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
3645 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
3646 kEncodeFinishDelayMs);
3647
3648 video_stream_encoder_->Stop();
3649}
perkj26091b12016-09-01 01:17:40 -07003650} // namespace webrtc