blob: 3b80f8544b73ac4240665311046aa02d363143d4 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080021#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010023#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020024#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020025#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010026#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020027#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070028#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080029#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020030#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "modules/video_coding/utility/default_video_bitrate_allocator.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010032#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010036#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020037#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "system_wrappers/include/sleep.h"
39#include "test/encoder_settings.h"
40#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020041#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "test/frame_generator.h"
43#include "test/gmock.h"
44#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020045#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070047
48namespace webrtc {
49
sprangb1ca0732017-02-01 08:38:12 -080050using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080051using ::testing::_;
kthelgason876222f2016-11-29 01:44:11 -080052
perkj803d97f2016-11-01 11:45:46 -070053namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020054const int kMinPixelsPerFrame = 320 * 180;
55const int kMinFramerateFps = 2;
56const int kMinBalancedFramerateFps = 7;
57const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080058const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010059const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020060const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010061const uint32_t kSimulcastTargetBitrateBps = 3150000;
62const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080063const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070064const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
asapersson5f7226f2016-11-25 04:37:00 -080066
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020067uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
68 0x00, 0x00, 0x03, 0x03, 0xF4,
69 0x05, 0x03, 0xC7, 0xE0, 0x1B,
70 0x41, 0x10, 0x8D, 0x00};
71
perkj803d97f2016-11-01 11:45:46 -070072class TestBuffer : public webrtc::I420Buffer {
73 public:
74 TestBuffer(rtc::Event* event, int width, int height)
75 : I420Buffer(width, height), event_(event) {}
76
77 private:
78 friend class rtc::RefCountedObject<TestBuffer>;
79 ~TestBuffer() override {
80 if (event_)
81 event_->Set();
82 }
83 rtc::Event* const event_;
84};
85
Noah Richards51db4212019-06-12 06:59:12 -070086// A fake native buffer that can't be converted to I420.
87class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
88 public:
89 FakeNativeBuffer(rtc::Event* event, int width, int height)
90 : event_(event), width_(width), height_(height) {}
91 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
92 int width() const override { return width_; }
93 int height() const override { return height_; }
94 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
95 return nullptr;
96 }
97
98 private:
99 friend class rtc::RefCountedObject<FakeNativeBuffer>;
100 ~FakeNativeBuffer() override {
101 if (event_)
102 event_->Set();
103 }
104 rtc::Event* const event_;
105 const int width_;
106 const int height_;
107};
108
Niels Möller7dc26b72017-12-06 10:27:48 +0100109class CpuOveruseDetectorProxy : public OveruseFrameDetector {
110 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200111 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
112 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100113 last_target_framerate_fps_(-1) {}
114 virtual ~CpuOveruseDetectorProxy() {}
115
116 void OnTargetFramerateUpdated(int framerate_fps) override {
117 rtc::CritScope cs(&lock_);
118 last_target_framerate_fps_ = framerate_fps;
119 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
120 }
121
122 int GetLastTargetFramerate() {
123 rtc::CritScope cs(&lock_);
124 return last_target_framerate_fps_;
125 }
126
Niels Möller4db138e2018-04-19 09:04:13 +0200127 CpuOveruseOptions GetOptions() { return options_; }
128
Niels Möller7dc26b72017-12-06 10:27:48 +0100129 private:
130 rtc::CriticalSection lock_;
131 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
132};
133
mflodmancc3d4422017-08-03 08:27:51 -0700134class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700135 public:
Niels Möller213618e2018-07-24 09:29:58 +0200136 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200137 const VideoStreamEncoderSettings& settings,
138 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100139 : VideoStreamEncoder(Clock::GetRealTimeClock(),
140 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200141 stats_proxy,
142 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200143 std::unique_ptr<OveruseFrameDetector>(
144 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100145 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200146 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700147
sprangb1ca0732017-02-01 08:38:12 -0800148 void PostTaskAndWait(bool down, AdaptReason reason) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100149 rtc::Event event;
kthelgason876222f2016-11-29 01:44:11 -0800150 encoder_queue()->PostTask([this, &event, reason, down] {
Åsa Perssonf5e5d252019-08-16 17:24:59 +0200151 if (down)
152 AdaptDown(reason);
153 else
154 AdaptUp(reason);
perkj803d97f2016-11-01 11:45:46 -0700155 event.Set();
156 });
perkj070ba852017-02-16 15:46:27 -0800157 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700158 }
159
kthelgason2fc52542017-03-03 00:24:41 -0800160 // This is used as a synchronisation mechanism, to make sure that the
161 // encoder queue is not blocked before we start sending it frames.
162 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100163 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200164 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800165 ASSERT_TRUE(event.Wait(5000));
166 }
167
sprangb1ca0732017-02-01 08:38:12 -0800168 void TriggerCpuOveruse() { PostTaskAndWait(true, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800169
sprangb1ca0732017-02-01 08:38:12 -0800170 void TriggerCpuNormalUsage() { PostTaskAndWait(false, AdaptReason::kCpu); }
kthelgason876222f2016-11-29 01:44:11 -0800171
sprangb1ca0732017-02-01 08:38:12 -0800172 void TriggerQualityLow() { PostTaskAndWait(true, AdaptReason::kQuality); }
kthelgason876222f2016-11-29 01:44:11 -0800173
sprangb1ca0732017-02-01 08:38:12 -0800174 void TriggerQualityHigh() { PostTaskAndWait(false, AdaptReason::kQuality); }
sprangfda496a2017-06-15 04:21:07 -0700175
Niels Möller7dc26b72017-12-06 10:27:48 +0100176 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700177};
178
asapersson5f7226f2016-11-25 04:37:00 -0800179class VideoStreamFactory
180 : public VideoEncoderConfig::VideoStreamFactoryInterface {
181 public:
sprangfda496a2017-06-15 04:21:07 -0700182 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
183 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800184 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700185 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800186 }
187
188 private:
189 std::vector<VideoStream> CreateEncoderStreams(
190 int width,
191 int height,
192 const VideoEncoderConfig& encoder_config) override {
193 std::vector<VideoStream> streams =
194 test::CreateVideoStreams(width, height, encoder_config);
195 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100196 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700197 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800198 }
199 return streams;
200 }
sprangfda496a2017-06-15 04:21:07 -0700201
asapersson5f7226f2016-11-25 04:37:00 -0800202 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700203 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800204};
205
Noah Richards51db4212019-06-12 06:59:12 -0700206// Simulates simulcast behavior and makes highest stream resolutions divisible
207// by 4.
208class CroppingVideoStreamFactory
209 : public VideoEncoderConfig::VideoStreamFactoryInterface {
210 public:
211 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
212 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
213 EXPECT_GT(num_temporal_layers, 0u);
214 EXPECT_GT(framerate, 0);
215 }
216
217 private:
218 std::vector<VideoStream> CreateEncoderStreams(
219 int width,
220 int height,
221 const VideoEncoderConfig& encoder_config) override {
222 std::vector<VideoStream> streams = test::CreateVideoStreams(
223 width - width % 4, height - height % 4, encoder_config);
224 for (VideoStream& stream : streams) {
225 stream.num_temporal_layers = num_temporal_layers_;
226 stream.max_framerate = framerate_;
227 }
228 return streams;
229 }
230
231 const size_t num_temporal_layers_;
232 const int framerate_;
233};
234
sprangb1ca0732017-02-01 08:38:12 -0800235class AdaptingFrameForwarder : public test::FrameForwarder {
236 public:
237 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700238 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800239
240 void set_adaptation_enabled(bool enabled) {
241 rtc::CritScope cs(&crit_);
242 adaptation_enabled_ = enabled;
243 }
244
asaperssonfab67072017-04-04 05:51:49 -0700245 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800246 rtc::CritScope cs(&crit_);
247 return adaptation_enabled_;
248 }
249
asapersson09f05612017-05-15 23:40:18 -0700250 rtc::VideoSinkWants last_wants() const {
251 rtc::CritScope cs(&crit_);
252 return last_wants_;
253 }
254
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200255 absl::optional<int> last_sent_width() const { return last_width_; }
256 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800257
sprangb1ca0732017-02-01 08:38:12 -0800258 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
259 int cropped_width = 0;
260 int cropped_height = 0;
261 int out_width = 0;
262 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700263 if (adaption_enabled()) {
264 if (adapter_.AdaptFrameResolution(
265 video_frame.width(), video_frame.height(),
266 video_frame.timestamp_us() * 1000, &cropped_width,
267 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100268 VideoFrame adapted_frame =
269 VideoFrame::Builder()
270 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
271 nullptr, out_width, out_height))
272 .set_timestamp_rtp(99)
273 .set_timestamp_ms(99)
274 .set_rotation(kVideoRotation_0)
275 .build();
sprangc5d62e22017-04-02 23:53:04 -0700276 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
277 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800278 last_width_.emplace(adapted_frame.width());
279 last_height_.emplace(adapted_frame.height());
280 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200281 last_width_ = absl::nullopt;
282 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700283 }
sprangb1ca0732017-02-01 08:38:12 -0800284 } else {
285 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800286 last_width_.emplace(video_frame.width());
287 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800288 }
289 }
290
291 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
292 const rtc::VideoSinkWants& wants) override {
293 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700294 last_wants_ = sink_wants();
sprangc5d62e22017-04-02 23:53:04 -0700295 adapter_.OnResolutionFramerateRequest(wants.target_pixel_count,
296 wants.max_pixel_count,
297 wants.max_framerate_fps);
sprangb1ca0732017-02-01 08:38:12 -0800298 test::FrameForwarder::AddOrUpdateSink(sink, wants);
299 }
sprangb1ca0732017-02-01 08:38:12 -0800300 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700301 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
302 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200303 absl::optional<int> last_width_;
304 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800305};
sprangc5d62e22017-04-02 23:53:04 -0700306
Niels Möller213618e2018-07-24 09:29:58 +0200307// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700308class MockableSendStatisticsProxy : public SendStatisticsProxy {
309 public:
310 MockableSendStatisticsProxy(Clock* clock,
311 const VideoSendStream::Config& config,
312 VideoEncoderConfig::ContentType content_type)
313 : SendStatisticsProxy(clock, config, content_type) {}
314
315 VideoSendStream::Stats GetStats() override {
316 rtc::CritScope cs(&lock_);
317 if (mock_stats_)
318 return *mock_stats_;
319 return SendStatisticsProxy::GetStats();
320 }
321
Niels Möller213618e2018-07-24 09:29:58 +0200322 int GetInputFrameRate() const override {
323 rtc::CritScope cs(&lock_);
324 if (mock_stats_)
325 return mock_stats_->input_frame_rate;
326 return SendStatisticsProxy::GetInputFrameRate();
327 }
sprangc5d62e22017-04-02 23:53:04 -0700328 void SetMockStats(const VideoSendStream::Stats& stats) {
329 rtc::CritScope cs(&lock_);
330 mock_stats_.emplace(stats);
331 }
332
333 void ResetMockStats() {
334 rtc::CritScope cs(&lock_);
335 mock_stats_.reset();
336 }
337
338 private:
339 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200340 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700341};
342
sprang4847ae62017-06-27 07:06:52 -0700343class MockBitrateObserver : public VideoBitrateAllocationObserver {
344 public:
Erik Språng566124a2018-04-23 12:32:22 +0200345 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700346};
347
perkj803d97f2016-11-01 11:45:46 -0700348} // namespace
349
mflodmancc3d4422017-08-03 08:27:51 -0700350class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700351 public:
352 static const int kDefaultTimeoutMs = 30 * 1000;
353
mflodmancc3d4422017-08-03 08:27:51 -0700354 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700355 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700356 codec_width_(320),
357 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200358 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200359 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700360 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200361 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700362 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700363 Clock::GetRealTimeClock(),
364 video_send_config_,
365 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700366 sink_(&fake_encoder_) {}
367
368 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700369 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700370 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200371 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800372 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200373 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200374 video_send_config_.rtp.payload_name = "FAKE";
375 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700376
Per512ecb32016-09-23 15:52:06 +0200377 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200378 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700379 video_encoder_config.video_stream_factory =
380 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100381 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700382
383 // Framerate limit is specified by the VideoStreamFactory.
384 std::vector<VideoStream> streams =
385 video_encoder_config.video_stream_factory->CreateEncoderStreams(
386 codec_width_, codec_height_, video_encoder_config);
387 max_framerate_ = streams[0].max_framerate;
Sebastian Jansson40889f32019-04-17 12:11:20 +0200388 fake_clock_.SetTime(Timestamp::us(1234));
sprang4847ae62017-06-27 07:06:52 -0700389
Niels Möllerf1338562018-04-26 09:51:47 +0200390 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800391 }
392
Niels Möllerf1338562018-04-26 09:51:47 +0200393 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700394 if (video_stream_encoder_)
395 video_stream_encoder_->Stop();
396 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200397 stats_proxy_.get(), video_send_config_.encoder_settings,
398 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700399 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
400 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700401 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700402 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
403 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200404 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700405 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800406 }
407
408 void ResetEncoder(const std::string& payload_name,
409 size_t num_streams,
410 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700411 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700412 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200413 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800414
415 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200416 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800417 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100418 video_encoder_config.max_bitrate_bps =
419 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800420 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700421 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
422 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700423 video_encoder_config.content_type =
424 screenshare ? VideoEncoderConfig::ContentType::kScreen
425 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700426 if (payload_name == "VP9") {
427 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
428 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
429 video_encoder_config.encoder_specific_settings =
430 new rtc::RefCountedObject<
431 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
432 }
Niels Möllerf1338562018-04-26 09:51:47 +0200433 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700434 }
435
sprang57c2fff2017-01-16 06:24:02 -0800436 VideoFrame CreateFrame(int64_t ntp_time_ms,
437 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100438 VideoFrame frame =
439 VideoFrame::Builder()
440 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
441 destruction_event, codec_width_, codec_height_))
442 .set_timestamp_rtp(99)
443 .set_timestamp_ms(99)
444 .set_rotation(kVideoRotation_0)
445 .build();
sprang57c2fff2017-01-16 06:24:02 -0800446 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700447 return frame;
448 }
449
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100450 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
451 rtc::Event* destruction_event,
452 int offset_x) const {
453 VideoFrame frame =
454 VideoFrame::Builder()
455 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
456 destruction_event, codec_width_, codec_height_))
457 .set_timestamp_rtp(99)
458 .set_timestamp_ms(99)
459 .set_rotation(kVideoRotation_0)
460 .set_update_rect({offset_x, 0, 1, 1})
461 .build();
462 frame.set_ntp_time_ms(ntp_time_ms);
463 return frame;
464 }
465
sprang57c2fff2017-01-16 06:24:02 -0800466 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100467 VideoFrame frame =
468 VideoFrame::Builder()
469 .set_video_frame_buffer(
470 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
471 .set_timestamp_rtp(99)
472 .set_timestamp_ms(99)
473 .set_rotation(kVideoRotation_0)
474 .build();
sprang57c2fff2017-01-16 06:24:02 -0800475 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700476 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700477 return frame;
478 }
479
Noah Richards51db4212019-06-12 06:59:12 -0700480 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
481 rtc::Event* destruction_event,
482 int width,
483 int height) const {
484 VideoFrame frame =
485 VideoFrame::Builder()
486 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
487 destruction_event, width, height))
488 .set_timestamp_rtp(99)
489 .set_timestamp_ms(99)
490 .set_rotation(kVideoRotation_0)
491 .build();
492 frame.set_ntp_time_ms(ntp_time_ms);
493 return frame;
494 }
495
496 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
497 rtc::Event* destruction_event) const {
498 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
499 codec_height_);
500 }
501
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100502 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
503 MockBitrateObserver bitrate_observer;
504 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
505
506 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
507 .Times(1);
508 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +0200509 DataRate::bps(kTargetBitrateBps), 0,
510 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100511
512 video_source_.IncomingCapturedFrame(
513 CreateFrame(1, codec_width_, codec_height_));
514 WaitForEncodedFrame(1);
515 }
516
asapersson02465b82017-04-10 01:12:52 -0700517 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700518 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700519 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
520 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700521 }
522
asapersson09f05612017-05-15 23:40:18 -0700523 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
524 const rtc::VideoSinkWants& wants2) {
525 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
526 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
527 }
528
Åsa Persson8c1bf952018-09-13 10:42:19 +0200529 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
530 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
531 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
532 EXPECT_FALSE(wants.target_pixel_count);
533 }
534
asapersson09f05612017-05-15 23:40:18 -0700535 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
536 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200537 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700538 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
539 EXPECT_GT(wants1.max_pixel_count, 0);
540 }
541
542 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
543 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200544 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700545 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
546 }
547
asaperssonf7e294d2017-06-13 23:25:22 -0700548 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
549 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200550 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700551 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
552 }
553
554 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
555 const rtc::VideoSinkWants& wants2) {
556 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
557 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
558 }
559
560 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
561 const rtc::VideoSinkWants& wants2) {
562 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
563 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
564 }
565
566 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
567 const rtc::VideoSinkWants& wants2) {
568 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
569 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
570 EXPECT_GT(wants1.max_pixel_count, 0);
571 }
572
573 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
574 const rtc::VideoSinkWants& wants2) {
575 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
576 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
577 }
578
asapersson09f05612017-05-15 23:40:18 -0700579 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
580 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200581 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700582 EXPECT_LT(wants.max_pixel_count, pixel_count);
583 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700584 }
585
586 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
587 EXPECT_LT(wants.max_framerate_fps, fps);
588 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
589 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700590 }
591
asaperssonf7e294d2017-06-13 23:25:22 -0700592 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
593 int expected_fps) {
594 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
595 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
596 EXPECT_FALSE(wants.target_pixel_count);
597 }
598
Jonathan Yubc771b72017-12-08 17:04:29 -0800599 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
600 int last_frame_pixels) {
601 // Balanced mode should always scale FPS to the desired range before
602 // attempting to scale resolution.
603 int fps_limit = wants.max_framerate_fps;
604 if (last_frame_pixels <= 320 * 240) {
605 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
606 } else if (last_frame_pixels <= 480 * 270) {
607 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
608 } else if (last_frame_pixels <= 640 * 480) {
609 EXPECT_LE(15, fps_limit);
610 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200611 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800612 }
613 }
614
sprang4847ae62017-06-27 07:06:52 -0700615 void WaitForEncodedFrame(int64_t expected_ntp_time) {
616 sink_.WaitForEncodedFrame(expected_ntp_time);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200617 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700618 }
619
620 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
621 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200622 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700623 return ok;
624 }
625
626 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
627 sink_.WaitForEncodedFrame(expected_width, expected_height);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200628 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700629 }
630
631 void ExpectDroppedFrame() {
632 sink_.ExpectDroppedFrame();
Sebastian Jansson40889f32019-04-17 12:11:20 +0200633 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700634 }
635
636 bool WaitForFrame(int64_t timeout_ms) {
637 bool ok = sink_.WaitForFrame(timeout_ms);
Sebastian Jansson40889f32019-04-17 12:11:20 +0200638 fake_clock_.AdvanceTime(TimeDelta::seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700639 return ok;
640 }
641
perkj26091b12016-09-01 01:17:40 -0700642 class TestEncoder : public test::FakeEncoder {
643 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100644 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700645
asaperssonfab67072017-04-04 05:51:49 -0700646 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800647 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700648 return config_;
649 }
650
651 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800652 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700653 block_next_encode_ = true;
654 }
655
Erik Språngaed30702018-11-05 12:57:17 +0100656 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800657 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100658 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100659 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100660 if (quality_scaling_) {
661 info.scaling_settings =
662 VideoEncoder::ScalingSettings(1, 2, kMinPixelsPerFrame);
663 }
664 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100665 for (int i = 0; i < kMaxSpatialLayers; ++i) {
666 if (temporal_layers_supported_[i]) {
667 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
668 info.fps_allocation[i].resize(num_layers);
669 }
670 }
Erik Språngaed30702018-11-05 12:57:17 +0100671 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200672
673 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Erik Språngaed30702018-11-05 12:57:17 +0100674 return info;
kthelgason876222f2016-11-29 01:44:11 -0800675 }
676
Erik Språngb7cb7b52019-02-26 15:52:33 +0100677 int32_t RegisterEncodeCompleteCallback(
678 EncodedImageCallback* callback) override {
679 rtc::CritScope lock(&local_crit_sect_);
680 encoded_image_callback_ = callback;
681 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
682 }
683
perkjfa10b552016-10-02 23:45:26 -0700684 void ContinueEncode() { continue_encode_event_.Set(); }
685
686 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
687 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800688 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700689 EXPECT_EQ(timestamp_, timestamp);
690 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
691 }
692
kthelgason2fc52542017-03-03 00:24:41 -0800693 void SetQualityScaling(bool b) {
694 rtc::CritScope lock(&local_crit_sect_);
695 quality_scaling_ = b;
696 }
kthelgasonad9010c2017-02-14 00:46:51 -0800697
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100698 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
699 rtc::CritScope lock(&local_crit_sect_);
700 is_hardware_accelerated_ = is_hardware_accelerated;
701 }
702
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100703 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
704 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
705 rtc::CritScope lock(&local_crit_sect_);
706 temporal_layers_supported_[spatial_idx] = supported;
707 }
708
Sergey Silkin6456e352019-07-08 17:56:40 +0200709 void SetResolutionBitrateLimits(
710 std::vector<ResolutionBitrateLimits> thresholds) {
711 rtc::CritScope cs(&local_crit_sect_);
712 resolution_bitrate_limits_ = thresholds;
713 }
714
sprangfe627f32017-03-29 08:24:59 -0700715 void ForceInitEncodeFailure(bool force_failure) {
716 rtc::CritScope lock(&local_crit_sect_);
717 force_init_encode_failed_ = force_failure;
718 }
719
Niels Möller6bb5ab92019-01-11 11:11:10 +0100720 void SimulateOvershoot(double rate_factor) {
721 rtc::CritScope lock(&local_crit_sect_);
722 rate_factor_ = rate_factor;
723 }
724
Erik Språngd7329ca2019-02-21 21:19:53 +0100725 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100726 rtc::CritScope lock(&local_crit_sect_);
727 return last_framerate_;
728 }
729
Erik Språngd7329ca2019-02-21 21:19:53 +0100730 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100731 rtc::CritScope lock(&local_crit_sect_);
732 return last_update_rect_;
733 }
734
Niels Möller87e2d782019-03-07 10:18:23 +0100735 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100736 rtc::CritScope lock(&local_crit_sect_);
737 return last_frame_types_;
738 }
739
740 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100741 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100742 keyframe ? VideoFrameType::kVideoFrameKey
743 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100744 {
745 rtc::CritScope lock(&local_crit_sect_);
746 last_frame_types_ = frame_type;
747 }
Niels Möllerb859b322019-03-07 12:40:01 +0100748 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100749 }
750
Erik Språngb7cb7b52019-02-26 15:52:33 +0100751 void InjectEncodedImage(const EncodedImage& image) {
752 rtc::CritScope lock(&local_crit_sect_);
753 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
754 }
755
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200756 void InjectEncodedImage(const EncodedImage& image,
757 const CodecSpecificInfo* codec_specific_info,
758 const RTPFragmentationHeader* fragmentation) {
759 rtc::CritScope lock(&local_crit_sect_);
760 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
761 fragmentation);
762 }
763
Erik Språngd7329ca2019-02-21 21:19:53 +0100764 void ExpectNullFrame() {
765 rtc::CritScope lock(&local_crit_sect_);
766 expect_null_frame_ = true;
767 }
768
Erik Språng5056af02019-09-02 15:53:11 +0200769 absl::optional<VideoEncoder::RateControlParameters>
770 GetAndResetLastRateControlSettings() {
771 auto settings = last_rate_control_settings_;
772 last_rate_control_settings_.reset();
773 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100774 }
775
Sergey Silkin5ee69672019-07-02 14:18:34 +0200776 int GetNumEncoderInitializations() const {
777 rtc::CritScope lock(&local_crit_sect_);
778 return num_encoder_initializations_;
779 }
780
perkjfa10b552016-10-02 23:45:26 -0700781 private:
perkj26091b12016-09-01 01:17:40 -0700782 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100783 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700784 bool block_encode;
785 {
brandtre78d2662017-01-16 05:57:16 -0800786 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100787 if (expect_null_frame_) {
788 EXPECT_EQ(input_image.timestamp(), 0u);
789 EXPECT_EQ(input_image.width(), 1);
790 last_frame_types_ = *frame_types;
791 expect_null_frame_ = false;
792 } else {
793 EXPECT_GT(input_image.timestamp(), timestamp_);
794 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
795 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
796 }
perkj26091b12016-09-01 01:17:40 -0700797
798 timestamp_ = input_image.timestamp();
799 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700800 last_input_width_ = input_image.width();
801 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700802 block_encode = block_next_encode_;
803 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100804 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100805 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700806 }
Niels Möllerb859b322019-03-07 12:40:01 +0100807 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700808 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700809 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700810 return result;
811 }
812
sprangfe627f32017-03-29 08:24:59 -0700813 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200814 const Settings& settings) override {
815 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200816
sprangfe627f32017-03-29 08:24:59 -0700817 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100818 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200819
820 ++num_encoder_initializations_;
821
Erik Språng82fad3d2018-03-21 09:57:23 +0100822 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700823 // Simulate setting up temporal layers, in order to validate the life
824 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100825 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200826 frame_buffer_controller_ =
827 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700828 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100829 if (force_init_encode_failed_) {
830 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700831 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100832 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100833
Erik Språngb7cb7b52019-02-26 15:52:33 +0100834 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700835 return res;
836 }
837
Erik Språngb7cb7b52019-02-26 15:52:33 +0100838 int32_t Release() override {
839 rtc::CritScope lock(&local_crit_sect_);
840 EXPECT_NE(initialized_, EncoderState::kUninitialized);
841 initialized_ = EncoderState::kUninitialized;
842 return FakeEncoder::Release();
843 }
844
Erik Språng16cb8f52019-04-12 13:59:09 +0200845 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100846 rtc::CritScope lock(&local_crit_sect_);
847 VideoBitrateAllocation adjusted_rate_allocation;
848 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
849 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200850 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100851 adjusted_rate_allocation.SetBitrate(
852 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200853 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100854 rate_factor_));
855 }
856 }
857 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200858 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200859 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200860 RateControlParameters adjusted_paramters = parameters;
861 adjusted_paramters.bitrate = adjusted_rate_allocation;
862 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100863 }
864
brandtre78d2662017-01-16 05:57:16 -0800865 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100866 enum class EncoderState {
867 kUninitialized,
868 kInitializationFailed,
869 kInitialized
870 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
871 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700872 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700873 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700874 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
875 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
876 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
877 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
878 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100879 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100880 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700881 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100882 absl::optional<bool>
883 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
884 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700885 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100886 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
887 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200888 absl::optional<VideoEncoder::RateControlParameters>
889 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100890 VideoFrame::UpdateRect last_update_rect_
891 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100892 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100893 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100894 EncodedImageCallback* encoded_image_callback_
895 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 11:20:09 +0200896 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200897 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +0200898 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
899 RTC_GUARDED_BY(local_crit_sect_);
perkj26091b12016-09-01 01:17:40 -0700900 };
901
mflodmancc3d4422017-08-03 08:27:51 -0700902 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700903 public:
904 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100905 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700906
perkj26091b12016-09-01 01:17:40 -0700907 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700908 EXPECT_TRUE(
909 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
910 }
911
912 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
913 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700914 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700915 if (!encoded_frame_event_.Wait(timeout_ms))
916 return false;
perkj26091b12016-09-01 01:17:40 -0700917 {
918 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800919 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700920 }
921 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700922 return true;
perkj26091b12016-09-01 01:17:40 -0700923 }
924
sprangb1ca0732017-02-01 08:38:12 -0800925 void WaitForEncodedFrame(uint32_t expected_width,
926 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700927 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100928 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700929 }
930
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100931 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700932 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800933 uint32_t width = 0;
934 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -0800935 {
936 rtc::CritScope lock(&crit_);
937 width = last_width_;
938 height = last_height_;
939 }
940 EXPECT_EQ(expected_height, height);
941 EXPECT_EQ(expected_width, width);
942 }
943
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +0200944 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
945 VideoRotation rotation;
946 {
947 rtc::CritScope lock(&crit_);
948 rotation = last_rotation_;
949 }
950 EXPECT_EQ(expected_rotation, rotation);
951 }
952
kthelgason2fc52542017-03-03 00:24:41 -0800953 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -0800954
sprangc5d62e22017-04-02 23:53:04 -0700955 bool WaitForFrame(int64_t timeout_ms) {
956 return encoded_frame_event_.Wait(timeout_ms);
957 }
958
perkj26091b12016-09-01 01:17:40 -0700959 void SetExpectNoFrames() {
960 rtc::CritScope lock(&crit_);
961 expect_frames_ = false;
962 }
963
asaperssonfab67072017-04-04 05:51:49 -0700964 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +0200965 rtc::CritScope lock(&crit_);
966 return number_of_reconfigurations_;
967 }
968
asaperssonfab67072017-04-04 05:51:49 -0700969 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +0200970 rtc::CritScope lock(&crit_);
971 return min_transmit_bitrate_bps_;
972 }
973
Erik Språngd7329ca2019-02-21 21:19:53 +0100974 void SetNumExpectedLayers(size_t num_layers) {
975 rtc::CritScope lock(&crit_);
976 num_expected_layers_ = num_layers;
977 }
978
Erik Språngb7cb7b52019-02-26 15:52:33 +0100979 int64_t GetLastCaptureTimeMs() const {
980 rtc::CritScope lock(&crit_);
981 return last_capture_time_ms_;
982 }
983
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200984 std::vector<uint8_t> GetLastEncodedImageData() {
985 rtc::CritScope lock(&crit_);
986 return std::move(last_encoded_image_data_);
987 }
988
989 RTPFragmentationHeader GetLastFragmentation() {
990 rtc::CritScope lock(&crit_);
991 return std::move(last_fragmentation_);
992 }
993
perkj26091b12016-09-01 01:17:40 -0700994 private:
sergeyu2cb155a2016-11-04 11:39:29 -0700995 Result OnEncodedImage(
996 const EncodedImage& encoded_image,
997 const CodecSpecificInfo* codec_specific_info,
998 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +0200999 rtc::CritScope lock(&crit_);
1000 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001001 last_encoded_image_data_ = std::vector<uint8_t>(
1002 encoded_image.data(), encoded_image.data() + encoded_image.size());
1003 if (fragmentation) {
1004 last_fragmentation_.CopyFrom(*fragmentation);
1005 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001006 uint32_t timestamp = encoded_image.Timestamp();
1007 if (last_timestamp_ != timestamp) {
1008 num_received_layers_ = 1;
1009 } else {
1010 ++num_received_layers_;
1011 }
1012 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001013 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001014 last_width_ = encoded_image._encodedWidth;
1015 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001016 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001017 if (num_received_layers_ == num_expected_layers_) {
1018 encoded_frame_event_.Set();
1019 }
sprangb1ca0732017-02-01 08:38:12 -08001020 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001021 }
1022
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001023 void OnEncoderConfigurationChanged(
1024 std::vector<VideoStream> streams,
1025 VideoEncoderConfig::ContentType content_type,
1026 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001027 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001028 ++number_of_reconfigurations_;
1029 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1030 }
1031
perkj26091b12016-09-01 01:17:40 -07001032 rtc::CriticalSection crit_;
1033 TestEncoder* test_encoder_;
1034 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001035 std::vector<uint8_t> last_encoded_image_data_;
1036 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001037 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001038 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001039 uint32_t last_height_ = 0;
1040 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001041 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001042 size_t num_expected_layers_ = 1;
1043 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001044 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001045 int number_of_reconfigurations_ = 0;
1046 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001047 };
1048
Sergey Silkin5ee69672019-07-02 14:18:34 +02001049 class VideoBitrateAllocatorProxyFactory
1050 : public VideoBitrateAllocatorFactory {
1051 public:
1052 VideoBitrateAllocatorProxyFactory()
1053 : bitrate_allocator_factory_(
1054 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1055
1056 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1057 const VideoCodec& codec) override {
1058 rtc::CritScope lock(&crit_);
1059 codec_config_ = codec;
1060 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1061 }
1062
1063 VideoCodec codec_config() const {
1064 rtc::CritScope lock(&crit_);
1065 return codec_config_;
1066 }
1067
1068 private:
1069 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1070
1071 rtc::CriticalSection crit_;
1072 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1073 };
1074
perkj26091b12016-09-01 01:17:40 -07001075 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001076 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001077 int codec_width_;
1078 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001079 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001080 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001081 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001082 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001083 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001084 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001085 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001086 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001087 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001088 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001089};
1090
mflodmancc3d4422017-08-03 08:27:51 -07001091TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001092 video_stream_encoder_->OnBitrateUpdated(
1093 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001094 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001095 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001096 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001097 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001098 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001099}
1100
mflodmancc3d4422017-08-03 08:27:51 -07001101TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001102 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001103 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001104 // The encoder will cache up to one frame for a short duration. Adding two
1105 // frames means that the first frame will be dropped and the second frame will
1106 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001107 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001108 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001109 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001110
Erik Språng4c6ca302019-04-08 15:14:01 +02001111 video_stream_encoder_->OnBitrateUpdated(
1112 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001113
Sebastian Janssona3177052018-04-10 13:05:49 +02001114 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001115 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001116 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1117
1118 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001119 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001120}
1121
mflodmancc3d4422017-08-03 08:27:51 -07001122TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001123 video_stream_encoder_->OnBitrateUpdated(
1124 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001125 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001126 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001127
Erik Språng4c6ca302019-04-08 15:14:01 +02001128 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01001129 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001130 // The encoder will cache up to one frame for a short duration. Adding two
1131 // frames means that the first frame will be dropped and the second frame will
1132 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001133 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001134 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001135
Erik Språng4c6ca302019-04-08 15:14:01 +02001136 video_stream_encoder_->OnBitrateUpdated(
1137 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001138 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001139 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1140 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001141 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001142}
1143
mflodmancc3d4422017-08-03 08:27:51 -07001144TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001145 video_stream_encoder_->OnBitrateUpdated(
1146 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001147 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001148 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001149
1150 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001151 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001152
perkja49cbd32016-09-16 07:53:41 -07001153 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001154 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001155 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001156}
1157
mflodmancc3d4422017-08-03 08:27:51 -07001158TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001159 video_stream_encoder_->OnBitrateUpdated(
1160 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001161
perkja49cbd32016-09-16 07:53:41 -07001162 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001163 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001164
mflodmancc3d4422017-08-03 08:27:51 -07001165 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001166 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001167 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001168 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1169 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001170}
1171
mflodmancc3d4422017-08-03 08:27:51 -07001172TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001173 video_stream_encoder_->OnBitrateUpdated(
1174 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj26091b12016-09-01 01:17:40 -07001175
1176 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001177 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001178 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001179 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1180 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001181 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1182 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001183 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001184 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001185
mflodmancc3d4422017-08-03 08:27:51 -07001186 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001187}
1188
Noah Richards51db4212019-06-12 06:59:12 -07001189TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1190 video_stream_encoder_->OnBitrateUpdated(
1191 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1192
1193 rtc::Event frame_destroyed_event;
1194 video_source_.IncomingCapturedFrame(
1195 CreateFakeNativeFrame(1, &frame_destroyed_event));
1196 ExpectDroppedFrame();
1197 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1198 video_stream_encoder_->Stop();
1199}
1200
1201TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1202 // Use the cropping factory.
1203 video_encoder_config_.video_stream_factory =
1204 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1205 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1206 kMaxPayloadLength);
1207 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1208
1209 // Capture a frame at codec_width_/codec_height_.
1210 video_stream_encoder_->OnBitrateUpdated(
1211 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1212 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1213 WaitForEncodedFrame(1);
1214 // The encoder will have been configured once.
1215 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1216 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1217 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1218
1219 // Now send in a fake frame that needs to be cropped as the width/height
1220 // aren't divisible by 4 (see CreateEncoderStreams above).
1221 rtc::Event frame_destroyed_event;
1222 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1223 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1224 ExpectDroppedFrame();
1225 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1226 video_stream_encoder_->Stop();
1227}
1228
mflodmancc3d4422017-08-03 08:27:51 -07001229TEST_F(VideoStreamEncoderTest,
1230 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001231 video_stream_encoder_->OnBitrateUpdated(
1232 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Per21d45d22016-10-30 21:37:57 +01001233 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001234
1235 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001236 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001237 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001238 // The encoder will have been configured once when the first frame is
1239 // received.
1240 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001241
1242 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001243 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001244 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001245 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001246 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001247
1248 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001249 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001250 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001251 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001252 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001253
mflodmancc3d4422017-08-03 08:27:51 -07001254 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001255}
1256
mflodmancc3d4422017-08-03 08:27:51 -07001257TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001258 video_stream_encoder_->OnBitrateUpdated(
1259 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001260
1261 // Capture a frame and wait for it to synchronize with the encoder thread.
1262 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001263 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001264 // The encoder will have been configured once.
1265 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001266 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1267 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1268
1269 codec_width_ *= 2;
1270 codec_height_ *= 2;
1271 // Capture a frame with a higher resolution and wait for it to synchronize
1272 // with the encoder thread.
1273 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001274 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001275 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1276 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001277 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001278
mflodmancc3d4422017-08-03 08:27:51 -07001279 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001280}
1281
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001282TEST_F(VideoStreamEncoderTest,
1283 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1284 video_stream_encoder_->OnBitrateUpdated(
1285 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1286
1287 // Capture a frame and wait for it to synchronize with the encoder thread.
1288 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1289 WaitForEncodedFrame(1);
1290
1291 VideoEncoderConfig video_encoder_config;
1292 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1293 // Changing the max payload data length recreates encoder.
1294 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1295 kMaxPayloadLength / 2);
1296
1297 // Capture a frame and wait for it to synchronize with the encoder thread.
1298 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1299 WaitForEncodedFrame(2);
1300 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1301
1302 video_stream_encoder_->Stop();
1303}
1304
Sergey Silkin5ee69672019-07-02 14:18:34 +02001305TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1306 video_stream_encoder_->OnBitrateUpdated(
1307 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1308
1309 VideoEncoderConfig video_encoder_config;
1310 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1311 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1312 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1313 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1314 kMaxPayloadLength);
1315
1316 // Capture a frame and wait for it to synchronize with the encoder thread.
1317 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1318 WaitForEncodedFrame(1);
1319 // The encoder will have been configured once when the first frame is
1320 // received.
1321 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1322 EXPECT_EQ(kTargetBitrateBps,
1323 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1324 EXPECT_EQ(kStartBitrateBps,
1325 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1326
Sergey Silkin6456e352019-07-08 17:56:40 +02001327 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1328 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001329 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1330 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1331 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1332 kMaxPayloadLength);
1333
1334 // Capture a frame and wait for it to synchronize with the encoder thread.
1335 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1336 WaitForEncodedFrame(2);
1337 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1338 // Bitrate limits have changed - rate allocator should be reconfigured,
1339 // encoder should not be reconfigured.
1340 EXPECT_EQ(kTargetBitrateBps * 2,
1341 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1342 EXPECT_EQ(kStartBitrateBps * 2,
1343 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1344 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1345
1346 video_stream_encoder_->Stop();
1347}
1348
Sergey Silkin6456e352019-07-08 17:56:40 +02001349TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001350 EncoderRecommendedBitrateLimitsDoNotOverrideAppBitrateLimits) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001351 video_stream_encoder_->OnBitrateUpdated(
1352 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1353
Sergey Silkin6456e352019-07-08 17:56:40 +02001354 VideoEncoderConfig video_encoder_config;
1355 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1356 video_encoder_config.max_bitrate_bps = 0;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001357 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001358 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1359 kMaxPayloadLength);
1360
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001361 video_source_.IncomingCapturedFrame(CreateFrame(1, 360, 180));
Sergey Silkin6456e352019-07-08 17:56:40 +02001362 WaitForEncodedFrame(1);
Sergey Silkin6456e352019-07-08 17:56:40 +02001363
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001364 // Get the default bitrate limits and use them as baseline for custom
1365 // application and encoder recommended limits.
1366 const uint32_t kDefaultMinBitrateKbps =
1367 bitrate_allocator_factory_.codec_config().minBitrate;
1368 const uint32_t kDefaultMaxBitrateKbps =
1369 bitrate_allocator_factory_.codec_config().maxBitrate;
1370 const uint32_t kEncMinBitrateKbps = kDefaultMinBitrateKbps * 2;
1371 const uint32_t kEncMaxBitrateKbps = kDefaultMaxBitrateKbps * 2;
1372 const uint32_t kAppMinBitrateKbps = kDefaultMinBitrateKbps * 3;
1373 const uint32_t kAppMaxBitrateKbps = kDefaultMaxBitrateKbps * 3;
1374
1375 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1376 codec_width_ * codec_height_, kEncMinBitrateKbps * 1000,
1377 kEncMinBitrateKbps * 1000, kEncMaxBitrateKbps * 1000);
1378 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1379
1380 // Change resolution. This will trigger encoder re-configuration and video
1381 // stream encoder will pick up the bitrate limits recommended by encoder.
1382 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1383 WaitForEncodedFrame(2);
1384 video_source_.IncomingCapturedFrame(CreateFrame(3, 360, 180));
1385 WaitForEncodedFrame(3);
1386
1387 // App bitrate limits are not set - bitrate limits recommended by encoder
1388 // should be used.
1389 EXPECT_EQ(kEncMaxBitrateKbps,
1390 bitrate_allocator_factory_.codec_config().maxBitrate);
1391 EXPECT_EQ(kEncMinBitrateKbps,
1392 bitrate_allocator_factory_.codec_config().minBitrate);
1393
1394 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1395 video_encoder_config.simulcast_layers[0].min_bitrate_bps = 0;
1396 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1397 kMaxPayloadLength);
1398 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1399 WaitForEncodedFrame(4);
1400
1401 // App limited the max bitrate - bitrate limits recommended by encoder should
1402 // not be applied.
1403 EXPECT_EQ(kAppMaxBitrateKbps,
1404 bitrate_allocator_factory_.codec_config().maxBitrate);
1405 EXPECT_EQ(kDefaultMinBitrateKbps,
1406 bitrate_allocator_factory_.codec_config().minBitrate);
1407
1408 video_encoder_config.max_bitrate_bps = 0;
1409 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1410 kAppMinBitrateKbps * 1000;
1411 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1412 kMaxPayloadLength);
1413 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1414 WaitForEncodedFrame(5);
1415
1416 // App limited the min bitrate - bitrate limits recommended by encoder should
1417 // not be applied.
1418 EXPECT_EQ(kDefaultMaxBitrateKbps,
1419 bitrate_allocator_factory_.codec_config().maxBitrate);
1420 EXPECT_EQ(kAppMinBitrateKbps,
1421 bitrate_allocator_factory_.codec_config().minBitrate);
1422
1423 video_encoder_config.max_bitrate_bps = kAppMaxBitrateKbps * 1000;
1424 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1425 kAppMinBitrateKbps * 1000;
Sergey Silkin6456e352019-07-08 17:56:40 +02001426 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1427 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001428 video_source_.IncomingCapturedFrame(CreateFrame(6, nullptr));
1429 WaitForEncodedFrame(6);
Sergey Silkin6456e352019-07-08 17:56:40 +02001430
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001431 // App limited both min and max bitrates - bitrate limits recommended by
1432 // encoder should not be applied.
1433 EXPECT_EQ(kAppMaxBitrateKbps,
1434 bitrate_allocator_factory_.codec_config().maxBitrate);
1435 EXPECT_EQ(kAppMinBitrateKbps,
1436 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001437
1438 video_stream_encoder_->Stop();
1439}
1440
1441TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001442 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001443 video_stream_encoder_->OnBitrateUpdated(
1444 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1445
1446 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001447 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001448 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001449 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001450 fake_encoder_.SetResolutionBitrateLimits(
1451 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1452
1453 VideoEncoderConfig video_encoder_config;
1454 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1455 video_encoder_config.max_bitrate_bps = 0;
1456 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1457 kMaxPayloadLength);
1458
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001459 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001460 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1461 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001462 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1463 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001464 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1465 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1466
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001467 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001468 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1469 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001470 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1471 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001472 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1473 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1474
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001475 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001476 // encoder for 360p should be used.
1477 video_source_.IncomingCapturedFrame(
1478 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1479 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001480 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1481 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001482 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1483 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1484
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001485 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001486 // ignored.
1487 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1488 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001489 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1490 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001491 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1492 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001493 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1494 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001495 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1496 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1497
1498 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1499 // for 270p should be used.
1500 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1501 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001502 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1503 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001504 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1505 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1506
1507 video_stream_encoder_->Stop();
1508}
1509
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001510TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1511 video_stream_encoder_->OnBitrateUpdated(
1512 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
1513
1514 VideoEncoderConfig video_encoder_config;
1515 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1516 video_encoder_config.max_bitrate_bps = 0;
1517 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1518 kMaxPayloadLength);
1519
1520 // Encode 720p frame to get the default encoder target bitrate.
1521 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1522 WaitForEncodedFrame(1);
1523 const uint32_t kDefaultTargetBitrateFor720pKbps =
1524 bitrate_allocator_factory_.codec_config()
1525 .simulcastStream[0]
1526 .targetBitrate;
1527
1528 // Set the max recommended encoder bitrate to something lower than the default
1529 // target bitrate.
1530 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1531 1280 * 720, 10 * 1000, 10 * 1000,
1532 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1533 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1534
1535 // Change resolution to trigger encoder reinitialization.
1536 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1537 WaitForEncodedFrame(2);
1538 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1539 WaitForEncodedFrame(3);
1540
1541 // Ensure the target bitrate is capped by the max bitrate.
1542 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1543 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1544 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1545 .simulcastStream[0]
1546 .targetBitrate *
1547 1000,
1548 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1549
1550 video_stream_encoder_->Stop();
1551}
1552
mflodmancc3d4422017-08-03 08:27:51 -07001553TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001554 EXPECT_TRUE(video_source_.has_sinks());
1555 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001556 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001557 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001558 EXPECT_FALSE(video_source_.has_sinks());
1559 EXPECT_TRUE(new_video_source.has_sinks());
1560
mflodmancc3d4422017-08-03 08:27:51 -07001561 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001562}
1563
mflodmancc3d4422017-08-03 08:27:51 -07001564TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001565 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001566 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001567 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001568 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001569}
1570
Jonathan Yubc771b72017-12-08 17:04:29 -08001571TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1572 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001573 const int kWidth = 1280;
1574 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001575
1576 // We rely on the automatic resolution adaptation, but we handle framerate
1577 // adaptation manually by mocking the stats proxy.
1578 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001579
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001580 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001581 video_stream_encoder_->OnBitrateUpdated(
1582 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001583 video_stream_encoder_->SetSource(&video_source_,
1584 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001585 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001586 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001587 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001588 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1589
Jonathan Yubc771b72017-12-08 17:04:29 -08001590 // Adapt down as far as possible.
1591 rtc::VideoSinkWants last_wants;
1592 int64_t t = 1;
1593 int loop_count = 0;
1594 do {
1595 ++loop_count;
1596 last_wants = video_source_.sink_wants();
1597
1598 // Simulate the framerate we've been asked to adapt to.
1599 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1600 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1601 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1602 mock_stats.input_frame_rate = fps;
1603 stats_proxy_->SetMockStats(mock_stats);
1604
1605 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1606 sink_.WaitForEncodedFrame(t);
1607 t += frame_interval_ms;
1608
mflodmancc3d4422017-08-03 08:27:51 -07001609 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001610 VerifyBalancedModeFpsRange(
1611 video_source_.sink_wants(),
1612 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1613 } while (video_source_.sink_wants().max_pixel_count <
1614 last_wants.max_pixel_count ||
1615 video_source_.sink_wants().max_framerate_fps <
1616 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001617
Jonathan Yubc771b72017-12-08 17:04:29 -08001618 // Verify that we've adapted all the way down.
1619 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001620 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001621 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1622 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001623 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001624 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1625 *video_source_.last_sent_height());
1626 EXPECT_EQ(kMinBalancedFramerateFps,
1627 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001628
Jonathan Yubc771b72017-12-08 17:04:29 -08001629 // Adapt back up the same number of times we adapted down.
1630 for (int i = 0; i < loop_count - 1; ++i) {
1631 last_wants = video_source_.sink_wants();
1632
1633 // Simulate the framerate we've been asked to adapt to.
1634 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1635 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1636 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1637 mock_stats.input_frame_rate = fps;
1638 stats_proxy_->SetMockStats(mock_stats);
1639
1640 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1641 sink_.WaitForEncodedFrame(t);
1642 t += frame_interval_ms;
1643
mflodmancc3d4422017-08-03 08:27:51 -07001644 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001645 VerifyBalancedModeFpsRange(
1646 video_source_.sink_wants(),
1647 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1648 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1649 last_wants.max_pixel_count ||
1650 video_source_.sink_wants().max_framerate_fps >
1651 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001652 }
1653
Åsa Persson8c1bf952018-09-13 10:42:19 +02001654 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001655 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001656 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001657 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1658 EXPECT_EQ((loop_count - 1) * 2,
1659 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001660
mflodmancc3d4422017-08-03 08:27:51 -07001661 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001662}
mflodmancc3d4422017-08-03 08:27:51 -07001663TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001664 video_stream_encoder_->OnBitrateUpdated(
1665 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001666 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001667
sprangc5d62e22017-04-02 23:53:04 -07001668 const int kFrameWidth = 1280;
1669 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001670
Åsa Persson8c1bf952018-09-13 10:42:19 +02001671 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001672
kthelgason5e13d412016-12-01 03:59:51 -08001673 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001674 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001675 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001676 frame_timestamp += kFrameIntervalMs;
1677
perkj803d97f2016-11-01 11:45:46 -07001678 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001679 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001680 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001681 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001682 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001683 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001684
asapersson0944a802017-04-07 00:57:58 -07001685 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001686 // wanted resolution.
1687 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1688 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1689 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001690 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001691
1692 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001693 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001694 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001695 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07001696
sprangc5d62e22017-04-02 23:53:04 -07001697 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001698 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001699
sprangc5d62e22017-04-02 23:53:04 -07001700 // Force an input frame rate to be available, or the adaptation call won't
1701 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001702 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001703 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001704 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001705 stats_proxy_->SetMockStats(stats);
1706
mflodmancc3d4422017-08-03 08:27:51 -07001707 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001708 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001709 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001710 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001711 frame_timestamp += kFrameIntervalMs;
1712
1713 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001714 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001715 EXPECT_EQ(std::numeric_limits<int>::max(),
1716 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001717 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001718
asapersson02465b82017-04-10 01:12:52 -07001719 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001720 video_stream_encoder_->SetSource(&new_video_source,
1721 webrtc::DegradationPreference::DISABLED);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001722 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001723
mflodmancc3d4422017-08-03 08:27:51 -07001724 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001725 new_video_source.IncomingCapturedFrame(
1726 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001727 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001728 frame_timestamp += kFrameIntervalMs;
1729
1730 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001731 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001732
1733 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001734 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001735 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
sprangc5d62e22017-04-02 23:53:04 -07001736 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1737 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001738 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001739 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001740
1741 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001742 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001743 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07001744 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1745 EXPECT_EQ(std::numeric_limits<int>::max(),
1746 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001747 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001748
mflodmancc3d4422017-08-03 08:27:51 -07001749 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001750}
1751
mflodmancc3d4422017-08-03 08:27:51 -07001752TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001753 video_stream_encoder_->OnBitrateUpdated(
1754 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001755
asaperssonfab67072017-04-04 05:51:49 -07001756 const int kWidth = 1280;
1757 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001758 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001759 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001760 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1761 EXPECT_FALSE(stats.bw_limited_resolution);
1762 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1763
1764 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001765 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001766 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001767 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001768
1769 stats = stats_proxy_->GetStats();
1770 EXPECT_TRUE(stats.bw_limited_resolution);
1771 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1772
1773 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001774 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001775 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001776 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001777
1778 stats = stats_proxy_->GetStats();
1779 EXPECT_FALSE(stats.bw_limited_resolution);
1780 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1781 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1782
mflodmancc3d4422017-08-03 08:27:51 -07001783 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001784}
1785
mflodmancc3d4422017-08-03 08:27:51 -07001786TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001787 video_stream_encoder_->OnBitrateUpdated(
1788 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001789
1790 const int kWidth = 1280;
1791 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001792 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001793 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001794 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1795 EXPECT_FALSE(stats.cpu_limited_resolution);
1796 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1797
1798 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001799 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001800 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001801 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001802
1803 stats = stats_proxy_->GetStats();
1804 EXPECT_TRUE(stats.cpu_limited_resolution);
1805 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1806
1807 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001808 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001809 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001810 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001811
1812 stats = stats_proxy_->GetStats();
1813 EXPECT_FALSE(stats.cpu_limited_resolution);
1814 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001815 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001816
mflodmancc3d4422017-08-03 08:27:51 -07001817 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001818}
1819
mflodmancc3d4422017-08-03 08:27:51 -07001820TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001821 video_stream_encoder_->OnBitrateUpdated(
1822 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001823
asaperssonfab67072017-04-04 05:51:49 -07001824 const int kWidth = 1280;
1825 const int kHeight = 720;
1826 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001827 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001828 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001829 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001830 EXPECT_FALSE(stats.cpu_limited_resolution);
1831 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1832
asaperssonfab67072017-04-04 05:51:49 -07001833 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001834 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001835 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001836 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001837 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001838 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001839 EXPECT_TRUE(stats.cpu_limited_resolution);
1840 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1841
1842 // Set new source with adaptation still enabled.
1843 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001844 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001845 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001846
asaperssonfab67072017-04-04 05:51:49 -07001847 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001848 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001849 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001850 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001851 EXPECT_TRUE(stats.cpu_limited_resolution);
1852 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1853
1854 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001855 video_stream_encoder_->SetSource(&new_video_source,
1856 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08001857
asaperssonfab67072017-04-04 05:51:49 -07001858 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001859 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001860 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001861 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001862 EXPECT_FALSE(stats.cpu_limited_resolution);
1863 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1864
1865 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07001866 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001867 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08001868
asaperssonfab67072017-04-04 05:51:49 -07001869 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001870 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001871 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001872 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001873 EXPECT_TRUE(stats.cpu_limited_resolution);
1874 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1875
asaperssonfab67072017-04-04 05:51:49 -07001876 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001877 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001878 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001879 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08001880 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001881 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001882 EXPECT_FALSE(stats.cpu_limited_resolution);
1883 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07001884 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001885
mflodmancc3d4422017-08-03 08:27:51 -07001886 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001887}
1888
mflodmancc3d4422017-08-03 08:27:51 -07001889TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001890 video_stream_encoder_->OnBitrateUpdated(
1891 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001892
asaperssonfab67072017-04-04 05:51:49 -07001893 const int kWidth = 1280;
1894 const int kHeight = 720;
1895 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001896 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001897 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001898 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001899 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001900 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001901
1902 // Set new source with adaptation still enabled.
1903 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001904 video_stream_encoder_->SetSource(&new_video_source,
1905 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001906
asaperssonfab67072017-04-04 05:51:49 -07001907 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001908 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001909 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001910 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001911 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001912 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001913
asaperssonfab67072017-04-04 05:51:49 -07001914 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001915 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001916 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001917 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08001918 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001919 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001920 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001921 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001922
asaperssonfab67072017-04-04 05:51:49 -07001923 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001924 video_stream_encoder_->SetSource(&new_video_source,
1925 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08001926
asaperssonfab67072017-04-04 05:51:49 -07001927 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001928 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08001929 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001930 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001931 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001932 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001933
asapersson02465b82017-04-10 01:12:52 -07001934 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07001935 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001936 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08001937
asaperssonfab67072017-04-04 05:51:49 -07001938 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001939 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08001940 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08001941 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07001942 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07001943 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1944 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08001945
mflodmancc3d4422017-08-03 08:27:51 -07001946 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08001947}
1948
mflodmancc3d4422017-08-03 08:27:51 -07001949TEST_F(VideoStreamEncoderTest,
1950 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001951 video_stream_encoder_->OnBitrateUpdated(
1952 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07001953
1954 const int kWidth = 1280;
1955 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001956 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07001957 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001958 video_source_.IncomingCapturedFrame(
1959 CreateFrame(timestamp_ms, kWidth, kHeight));
1960 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001961 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1962 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
1963 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1964
1965 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001966 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001967 timestamp_ms += kFrameIntervalMs;
1968 video_source_.IncomingCapturedFrame(
1969 CreateFrame(timestamp_ms, kWidth, kHeight));
1970 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001971 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
1972 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1973 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1974
1975 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001976 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02001977 timestamp_ms += kFrameIntervalMs;
1978 video_source_.IncomingCapturedFrame(
1979 CreateFrame(timestamp_ms, kWidth, kHeight));
1980 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001981 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1982 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
1983 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1984
Niels Möller4db138e2018-04-19 09:04:13 +02001985 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07001986 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02001987
1988 VideoEncoderConfig video_encoder_config;
1989 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1990 // Make format different, to force recreation of encoder.
1991 video_encoder_config.video_format.parameters["foo"] = "foo";
1992 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001993 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001994 timestamp_ms += kFrameIntervalMs;
1995 video_source_.IncomingCapturedFrame(
1996 CreateFrame(timestamp_ms, kWidth, kHeight));
1997 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07001998 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
1999 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2000 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2001
mflodmancc3d4422017-08-03 08:27:51 -07002002 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002003}
2004
mflodmancc3d4422017-08-03 08:27:51 -07002005TEST_F(VideoStreamEncoderTest,
2006 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002007 video_stream_encoder_->OnBitrateUpdated(
2008 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002009
asapersson0944a802017-04-07 00:57:58 -07002010 const int kWidth = 1280;
2011 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002012 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002013
asaperssonfab67072017-04-04 05:51:49 -07002014 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002015 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002016 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002017 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002018 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002019 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2020
asapersson02465b82017-04-10 01:12:52 -07002021 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002022 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002023 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002024 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002025 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002026 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002027 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002028 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2029
2030 // Set new source with adaptation still enabled.
2031 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002032 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002033 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002034
2035 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002036 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002037 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002038 stats = stats_proxy_->GetStats();
2039 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002040 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002041 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2042
sprangc5d62e22017-04-02 23:53:04 -07002043 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002044 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002045 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002046 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002047 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002048 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002049 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002050 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002051 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002052 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002053 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2054
sprangc5d62e22017-04-02 23:53:04 -07002055 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002056 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002057 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2058 mock_stats.input_frame_rate = 30;
2059 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002060 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002061 stats_proxy_->ResetMockStats();
2062
2063 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002064 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002065 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002066
2067 // Framerate now adapted.
2068 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002069 EXPECT_FALSE(stats.cpu_limited_resolution);
2070 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002071 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2072
2073 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002074 video_stream_encoder_->SetSource(&new_video_source,
2075 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002076 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002077 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002078 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002079
2080 stats = stats_proxy_->GetStats();
2081 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002082 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002083 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2084
2085 // Try to trigger overuse. Should not succeed.
2086 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002087 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002088 stats_proxy_->ResetMockStats();
2089
2090 stats = stats_proxy_->GetStats();
2091 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002092 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002093 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2094
2095 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002096 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002097 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002098 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002099 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002100 stats = stats_proxy_->GetStats();
2101 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002102 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002103 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002104
2105 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002106 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002107 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002108 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002109 stats = stats_proxy_->GetStats();
2110 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002111 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002112 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2113
2114 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002115 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002116 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002117 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002118 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002119 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002120 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002121 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002122 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002123 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002124 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2125
2126 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002127 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002128 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002129 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002130 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002131 stats = stats_proxy_->GetStats();
2132 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002133 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002134 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002135 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002136
mflodmancc3d4422017-08-03 08:27:51 -07002137 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002138}
2139
mflodmancc3d4422017-08-03 08:27:51 -07002140TEST_F(VideoStreamEncoderTest,
2141 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002142 const int kWidth = 1280;
2143 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002144 video_stream_encoder_->OnBitrateUpdated(
2145 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002146
asaperssonfab67072017-04-04 05:51:49 -07002147 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002148 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002149
asaperssonfab67072017-04-04 05:51:49 -07002150 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002151 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002152
asaperssonfab67072017-04-04 05:51:49 -07002153 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002154 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002155
asaperssonfab67072017-04-04 05:51:49 -07002156 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002157 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002158
kthelgason876222f2016-11-29 01:44:11 -08002159 // Expect a scale down.
2160 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002161 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002162
asapersson02465b82017-04-10 01:12:52 -07002163 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002164 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002165 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002166 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002167
asaperssonfab67072017-04-04 05:51:49 -07002168 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002169 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002170 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002171 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002172
asaperssonfab67072017-04-04 05:51:49 -07002173 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002174 EXPECT_EQ(std::numeric_limits<int>::max(),
2175 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002176
asaperssonfab67072017-04-04 05:51:49 -07002177 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002178 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002179 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002180 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002181
asapersson02465b82017-04-10 01:12:52 -07002182 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002183 EXPECT_EQ(std::numeric_limits<int>::max(),
2184 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002185
mflodmancc3d4422017-08-03 08:27:51 -07002186 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002187}
2188
mflodmancc3d4422017-08-03 08:27:51 -07002189TEST_F(VideoStreamEncoderTest,
2190 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002191 const int kWidth = 1280;
2192 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002193 video_stream_encoder_->OnBitrateUpdated(
2194 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002195
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002196 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002197 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002198 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002199 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002200
2201 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002202 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002203 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002204 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2205 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2206
2207 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002208 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002209 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002210 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2211 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2212 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2213
2214 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002215 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002216 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2217 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2218 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2219
mflodmancc3d4422017-08-03 08:27:51 -07002220 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002221}
2222
mflodmancc3d4422017-08-03 08:27:51 -07002223TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002224 const int kWidth = 1280;
2225 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002226 video_stream_encoder_->OnBitrateUpdated(
2227 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002228
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002229 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002230 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002231 video_stream_encoder_->SetSource(&source,
2232 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002233 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2234 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002235 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002236
2237 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002238 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002239 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2240 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2241 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2242 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2243
2244 // Trigger adapt down for same input resolution, expect no change.
2245 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2246 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002247 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002248 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2249 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2250 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2251
2252 // Trigger adapt down for larger input resolution, expect no change.
2253 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2254 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002255 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002256 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2257 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2258 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2259
mflodmancc3d4422017-08-03 08:27:51 -07002260 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002261}
2262
mflodmancc3d4422017-08-03 08:27:51 -07002263TEST_F(VideoStreamEncoderTest,
2264 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002265 const int kWidth = 1280;
2266 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002267 video_stream_encoder_->OnBitrateUpdated(
2268 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002269
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002270 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002271 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002272 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002273 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002274
2275 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002276 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002277 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002278 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2279 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2280
2281 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002282 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002283 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002284 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2285 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2286
mflodmancc3d4422017-08-03 08:27:51 -07002287 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002288}
2289
mflodmancc3d4422017-08-03 08:27:51 -07002290TEST_F(VideoStreamEncoderTest,
2291 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002292 const int kWidth = 1280;
2293 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002294 video_stream_encoder_->OnBitrateUpdated(
2295 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002296
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002297 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002298 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002299 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002300 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002301
2302 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002303 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002304 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002305 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002306 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2307
2308 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002309 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002310 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002311 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002312 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2313
mflodmancc3d4422017-08-03 08:27:51 -07002314 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002315}
2316
mflodmancc3d4422017-08-03 08:27:51 -07002317TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002318 const int kWidth = 1280;
2319 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002320 video_stream_encoder_->OnBitrateUpdated(
2321 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002322
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002323 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002324 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002325 video_stream_encoder_->SetSource(&source,
2326 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002327
2328 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2329 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002330 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002331 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2332 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2333 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2334
2335 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002336 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002337 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002338 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2341
mflodmancc3d4422017-08-03 08:27:51 -07002342 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002343}
2344
mflodmancc3d4422017-08-03 08:27:51 -07002345TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002346 const int kWidth = 1280;
2347 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002348 video_stream_encoder_->OnBitrateUpdated(
2349 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002350
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002351 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002352 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002353 video_stream_encoder_->SetSource(&source,
2354 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002355
2356 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2357 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002358 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002359 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2360 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2361 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2362
2363 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002364 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002365 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002366 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2367 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2368 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2369
mflodmancc3d4422017-08-03 08:27:51 -07002370 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002371}
2372
mflodmancc3d4422017-08-03 08:27:51 -07002373TEST_F(VideoStreamEncoderTest,
2374 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002375 const int kWidth = 1280;
2376 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002377 video_stream_encoder_->OnBitrateUpdated(
2378 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002379
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002380 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002381 AdaptingFrameForwarder source;
2382 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002383 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002384 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002385
2386 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002387 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002388 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002389 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2390 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2391
2392 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002393 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002394 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002395 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002396 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002397 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2398 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2399
2400 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002401 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002402 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2404 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2406
mflodmancc3d4422017-08-03 08:27:51 -07002407 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002408}
2409
mflodmancc3d4422017-08-03 08:27:51 -07002410TEST_F(VideoStreamEncoderTest,
2411 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002412 const int kWidth = 1280;
2413 const int kHeight = 720;
2414 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002415 video_stream_encoder_->OnBitrateUpdated(
2416 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002417
2418 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2419 stats.input_frame_rate = kInputFps;
2420 stats_proxy_->SetMockStats(stats);
2421
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002422 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002423 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2424 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002425 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002426
2427 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002428 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002429 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2430 sink_.WaitForEncodedFrame(2);
2431 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2432
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002433 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002434 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002435 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002436 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002437 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002438
2439 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002441 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2442 sink_.WaitForEncodedFrame(3);
2443 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2444
2445 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002446 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002447 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002448
mflodmancc3d4422017-08-03 08:27:51 -07002449 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002450}
2451
mflodmancc3d4422017-08-03 08:27:51 -07002452TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002453 const int kWidth = 1280;
2454 const int kHeight = 720;
2455 const size_t kNumFrames = 10;
2456
Erik Språng4c6ca302019-04-08 15:14:01 +02002457 video_stream_encoder_->OnBitrateUpdated(
2458 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002459
asaperssond0de2952017-04-21 01:47:31 -07002460 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002461 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002462 video_source_.set_adaptation_enabled(true);
2463
2464 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2465 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2466
2467 int downscales = 0;
2468 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002469 video_source_.IncomingCapturedFrame(
2470 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2471 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002472
asaperssonfab67072017-04-04 05:51:49 -07002473 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002474 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002475 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002476 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002477
2478 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2479 ++downscales;
2480
2481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2482 EXPECT_EQ(downscales,
2483 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2484 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002485 }
mflodmancc3d4422017-08-03 08:27:51 -07002486 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002487}
2488
mflodmancc3d4422017-08-03 08:27:51 -07002489TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002490 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2491 const int kWidth = 1280;
2492 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002493 video_stream_encoder_->OnBitrateUpdated(
2494 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002495
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002496 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002497 AdaptingFrameForwarder source;
2498 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002499 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002500 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002501
Åsa Persson8c1bf952018-09-13 10:42:19 +02002502 int64_t timestamp_ms = kFrameIntervalMs;
2503 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002504 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002505 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002506 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2507 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2508
2509 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002510 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002511 timestamp_ms += kFrameIntervalMs;
2512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2513 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002514 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002515 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2516 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2517
2518 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002519 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002520 timestamp_ms += kFrameIntervalMs;
2521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002522 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002523 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002524 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2525 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2526
2527 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002528 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002529 timestamp_ms += kFrameIntervalMs;
2530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2531 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002532 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002533 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2534 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2535
2536 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002537 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002538 timestamp_ms += kFrameIntervalMs;
2539 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002540 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002541 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002542 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2543 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2544
mflodmancc3d4422017-08-03 08:27:51 -07002545 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002546}
2547
mflodmancc3d4422017-08-03 08:27:51 -07002548TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002549 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2550 const int kWidth = 1280;
2551 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002552 video_stream_encoder_->OnBitrateUpdated(
2553 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002554
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002555 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002556 AdaptingFrameForwarder source;
2557 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002558 video_stream_encoder_->SetSource(&source,
2559 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002560
Åsa Persson8c1bf952018-09-13 10:42:19 +02002561 int64_t timestamp_ms = kFrameIntervalMs;
2562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002563 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002564 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2566 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2567
2568 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002569 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002570 timestamp_ms += kFrameIntervalMs;
2571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2572 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002573 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2575 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2576
2577 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002578 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002579 timestamp_ms += kFrameIntervalMs;
2580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002581 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002582 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002583 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2584 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2585
2586 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002587 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002588 timestamp_ms += kFrameIntervalMs;
2589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2590 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002591 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2593 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2594
2595 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002596 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002597 timestamp_ms += kFrameIntervalMs;
2598 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002599 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002600 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2602 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2603
mflodmancc3d4422017-08-03 08:27:51 -07002604 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002605}
2606
Åsa Persson1b247f12019-08-14 17:26:39 +02002607TEST_F(VideoStreamEncoderTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
2608 webrtc::test::ScopedFieldTrials field_trials(
2609 "WebRTC-Video-BalancedDegradationSettings/"
2610 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
2611 // Reset encoder for field trials to take effect.
2612 ConfigureEncoder(video_encoder_config_.Copy());
2613
2614 const int kWidth = 640; // pixels:640x360=230400
2615 const int kHeight = 360;
2616 const int64_t kFrameIntervalMs = 150;
2617 const int kMinBitrateBps = 425000;
2618 const int kTooLowMinBitrateBps = 424000;
2619 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTooLowMinBitrateBps),
2620 DataRate::bps(kTooLowMinBitrateBps),
2621 0, 0);
2622
2623 // Enable BALANCED preference, no initial limitation.
2624 AdaptingFrameForwarder source;
2625 source.set_adaptation_enabled(true);
2626 video_stream_encoder_->SetSource(&source,
2627 webrtc::DegradationPreference::BALANCED);
2628
2629 int64_t timestamp_ms = kFrameIntervalMs;
2630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2631 sink_.WaitForEncodedFrame(kWidth, kHeight);
2632 VerifyFpsMaxResolutionMax(source.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002633 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2634
2635 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2636 video_stream_encoder_->TriggerQualityLow();
2637 timestamp_ms += kFrameIntervalMs;
2638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2639 sink_.WaitForEncodedFrame(timestamp_ms);
2640 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02002641 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2642
2643 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2644 video_stream_encoder_->TriggerQualityLow();
2645 timestamp_ms += kFrameIntervalMs;
2646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2647 sink_.WaitForEncodedFrame(timestamp_ms);
2648 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002649 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2650
Åsa Persson30ab0152019-08-27 12:22:33 +02002651 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2652 video_stream_encoder_->TriggerQualityLow();
2653 timestamp_ms += kFrameIntervalMs;
2654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2655 sink_.WaitForEncodedFrame(timestamp_ms);
2656 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2657 EXPECT_EQ(source.sink_wants().max_framerate_fps, 10);
2658 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2659
2660 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02002661 video_stream_encoder_->TriggerQualityHigh();
2662 timestamp_ms += kFrameIntervalMs;
2663 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2664 sink_.WaitForEncodedFrame(timestamp_ms);
Åsa Persson30ab0152019-08-27 12:22:33 +02002665 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02002666
Åsa Persson30ab0152019-08-27 12:22:33 +02002667 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02002668 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
2669 DataRate::bps(kMinBitrateBps), 0, 0);
2670 video_stream_encoder_->TriggerQualityHigh();
2671 timestamp_ms += kFrameIntervalMs;
2672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2673 sink_.WaitForEncodedFrame(timestamp_ms);
Åsa Persson30ab0152019-08-27 12:22:33 +02002674 EXPECT_EQ(source.sink_wants().max_framerate_fps, 14);
2675 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2676
2677 video_stream_encoder_->Stop();
2678}
2679
2680TEST_F(VideoStreamEncoderTest,
2681 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
2682 webrtc::test::ScopedFieldTrials field_trials(
2683 "WebRTC-Video-BalancedDegradationSettings/"
2684 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
2685 // Reset encoder for field trials to take effect.
2686 ConfigureEncoder(video_encoder_config_.Copy());
2687
2688 const int kWidth = 640; // pixels:640x360=230400
2689 const int kHeight = 360;
2690 const int64_t kFrameIntervalMs = 150;
2691 const int kResolutionMinBitrateBps = 435000;
2692 const int kTooLowMinResolutionBitrateBps = 434000;
2693 video_stream_encoder_->OnBitrateUpdated(
2694 DataRate::bps(kTooLowMinResolutionBitrateBps),
2695 DataRate::bps(kTooLowMinResolutionBitrateBps), 0, 0);
2696
2697 // Enable BALANCED preference, no initial limitation.
2698 AdaptingFrameForwarder source;
2699 source.set_adaptation_enabled(true);
2700 video_stream_encoder_->SetSource(&source,
2701 webrtc::DegradationPreference::BALANCED);
2702
2703 int64_t timestamp_ms = kFrameIntervalMs;
2704 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2705 sink_.WaitForEncodedFrame(kWidth, kHeight);
2706 VerifyFpsMaxResolutionMax(source.sink_wants());
2707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2708
2709 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2710 video_stream_encoder_->TriggerQualityLow();
2711 timestamp_ms += kFrameIntervalMs;
2712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2713 sink_.WaitForEncodedFrame(timestamp_ms);
2714 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2715 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2716
2717 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2718 video_stream_encoder_->TriggerQualityLow();
2719 timestamp_ms += kFrameIntervalMs;
2720 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2721 sink_.WaitForEncodedFrame(timestamp_ms);
2722 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2723 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2724
2725 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2726 video_stream_encoder_->TriggerQualityLow();
2727 timestamp_ms += kFrameIntervalMs;
2728 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2729 sink_.WaitForEncodedFrame(timestamp_ms);
2730 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002731 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2732
Åsa Persson30ab0152019-08-27 12:22:33 +02002733 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
2734 video_stream_encoder_->TriggerQualityHigh();
2735 timestamp_ms += kFrameIntervalMs;
2736 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2737 sink_.WaitForEncodedFrame(timestamp_ms);
2738 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2739 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2740
2741 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2742 video_stream_encoder_->TriggerQualityHigh();
2743 timestamp_ms += kFrameIntervalMs;
2744 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2745 sink_.WaitForEncodedFrame(timestamp_ms);
2746 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2747
2748 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
2749 video_stream_encoder_->OnBitrateUpdated(
2750 DataRate::bps(kResolutionMinBitrateBps),
2751 DataRate::bps(kResolutionMinBitrateBps), 0, 0);
2752 video_stream_encoder_->TriggerQualityHigh();
2753 timestamp_ms += kFrameIntervalMs;
2754 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2755 sink_.WaitForEncodedFrame(timestamp_ms);
2756 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2757 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2758
2759 video_stream_encoder_->Stop();
2760}
2761
2762TEST_F(VideoStreamEncoderTest,
2763 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
2764 webrtc::test::ScopedFieldTrials field_trials(
2765 "WebRTC-Video-BalancedDegradationSettings/"
2766 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
2767 // Reset encoder for field trials to take effect.
2768 ConfigureEncoder(video_encoder_config_.Copy());
2769
2770 const int kWidth = 640; // pixels:640x360=230400
2771 const int kHeight = 360;
2772 const int64_t kFrameIntervalMs = 150;
2773 const int kMinBitrateBps = 425000;
2774 const int kTooLowMinBitrateBps = 424000;
2775 const int kResolutionMinBitrateBps = 435000;
2776 const int kTooLowMinResolutionBitrateBps = 434000;
2777 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kTooLowMinBitrateBps),
2778 DataRate::bps(kTooLowMinBitrateBps),
2779 0, 0);
2780
2781 // Enable BALANCED preference, no initial limitation.
2782 AdaptingFrameForwarder source;
2783 source.set_adaptation_enabled(true);
2784 video_stream_encoder_->SetSource(&source,
2785 webrtc::DegradationPreference::BALANCED);
2786
2787 int64_t timestamp_ms = kFrameIntervalMs;
2788 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2789 sink_.WaitForEncodedFrame(kWidth, kHeight);
2790 VerifyFpsMaxResolutionMax(source.sink_wants());
2791 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2792
2793 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2794 video_stream_encoder_->TriggerQualityLow();
2795 timestamp_ms += kFrameIntervalMs;
2796 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2797 sink_.WaitForEncodedFrame(timestamp_ms);
2798 VerifyFpsEqResolutionMax(source.sink_wants(), 14);
2799 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2800
2801 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2802 video_stream_encoder_->TriggerQualityLow();
2803 timestamp_ms += kFrameIntervalMs;
2804 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2805 sink_.WaitForEncodedFrame(timestamp_ms);
2806 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
2807 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2808
2809 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2810 video_stream_encoder_->TriggerQualityLow();
2811 timestamp_ms += kFrameIntervalMs;
2812 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2813 sink_.WaitForEncodedFrame(timestamp_ms);
2814 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
2815 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2816
2817 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
2818 video_stream_encoder_->TriggerQualityHigh();
2819 timestamp_ms += kFrameIntervalMs;
2820 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2821 sink_.WaitForEncodedFrame(timestamp_ms);
2822 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2823
2824 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
2825 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kMinBitrateBps),
2826 DataRate::bps(kMinBitrateBps), 0, 0);
2827 video_stream_encoder_->TriggerQualityHigh();
2828 timestamp_ms += kFrameIntervalMs;
2829 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2830 sink_.WaitForEncodedFrame(timestamp_ms);
2831 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
2832 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2833
2834 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
2835 video_stream_encoder_->OnBitrateUpdated(
2836 DataRate::bps(kTooLowMinResolutionBitrateBps),
2837 DataRate::bps(kTooLowMinResolutionBitrateBps), 0, 0);
2838 video_stream_encoder_->TriggerQualityHigh();
2839 timestamp_ms += kFrameIntervalMs;
2840 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2841 sink_.WaitForEncodedFrame(timestamp_ms);
2842 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2843
2844 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
2845 video_stream_encoder_->OnBitrateUpdated(
2846 DataRate::bps(kResolutionMinBitrateBps),
2847 DataRate::bps(kResolutionMinBitrateBps), 0, 0);
2848 video_stream_encoder_->TriggerQualityHigh();
2849 timestamp_ms += kFrameIntervalMs;
2850 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2851 sink_.WaitForEncodedFrame(timestamp_ms);
2852 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
2853 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2854
Åsa Persson1b247f12019-08-14 17:26:39 +02002855 video_stream_encoder_->Stop();
2856}
2857
mflodmancc3d4422017-08-03 08:27:51 -07002858TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002859 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
2860 const int kWidth = 1280;
2861 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002862 video_stream_encoder_->OnBitrateUpdated(
2863 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002864
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002865 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002866 AdaptingFrameForwarder source;
2867 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002868 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002869 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002870
Åsa Persson8c1bf952018-09-13 10:42:19 +02002871 int64_t timestamp_ms = kFrameIntervalMs;
2872 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002874 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002875 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2876 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2877 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2878 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2879
2880 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002881 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002882 timestamp_ms += kFrameIntervalMs;
2883 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2884 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002885 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002886 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2887 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2888 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2889 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2890
2891 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07002892 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002893 timestamp_ms += kFrameIntervalMs;
2894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2895 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002896 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002897 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2899 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2900 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2901
Jonathan Yubc771b72017-12-08 17:04:29 -08002902 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002903 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002904 timestamp_ms += kFrameIntervalMs;
2905 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2906 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002907 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002908 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2909 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002910 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002911 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2912
Jonathan Yubc771b72017-12-08 17:04:29 -08002913 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07002914 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002915 timestamp_ms += kFrameIntervalMs;
2916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2917 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002918 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002919 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07002920 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2921 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2922 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2923 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2924
Jonathan Yubc771b72017-12-08 17:04:29 -08002925 // Trigger quality adapt down, expect no change (min resolution reached).
2926 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002927 timestamp_ms += kFrameIntervalMs;
2928 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2929 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002930 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
2931 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2932 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2933 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2934 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2935
2936 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07002937 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002938 timestamp_ms += kFrameIntervalMs;
2939 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2940 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002941 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08002942 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2943 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2944 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2945 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2946
2947 // Trigger cpu adapt up, expect upscaled resolution (640x360).
2948 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002949 timestamp_ms += kFrameIntervalMs;
2950 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2951 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002952 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
2953 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2954 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2955 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2956 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2957
2958 // Trigger cpu adapt up, expect upscaled resolution (960x540).
2959 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002960 timestamp_ms += kFrameIntervalMs;
2961 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2962 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08002963 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07002964 last_wants = source.sink_wants();
2965 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2966 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002967 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002968 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2969
2970 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07002971 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002972 timestamp_ms += kFrameIntervalMs;
2973 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2974 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002975 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07002976 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2977 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002978 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002979 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2980
2981 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07002982 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002983 timestamp_ms += kFrameIntervalMs;
2984 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002985 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07002986 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02002987 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002988 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2989 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002990 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07002991 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08002992
mflodmancc3d4422017-08-03 08:27:51 -07002993 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08002994}
2995
mflodmancc3d4422017-08-03 08:27:51 -07002996TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07002997 const int kWidth = 640;
2998 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07002999
Erik Språng4c6ca302019-04-08 15:14:01 +02003000 video_stream_encoder_->OnBitrateUpdated(
3001 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003002
perkj803d97f2016-11-01 11:45:46 -07003003 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003004 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003005 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003006 }
3007
mflodmancc3d4422017-08-03 08:27:51 -07003008 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003009 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003010 video_source_.IncomingCapturedFrame(CreateFrame(
3011 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003012 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003013 }
3014
mflodmancc3d4422017-08-03 08:27:51 -07003015 video_stream_encoder_->Stop();
3016 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003017 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003018
perkj803d97f2016-11-01 11:45:46 -07003019 EXPECT_EQ(1,
3020 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3021 EXPECT_EQ(
3022 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3023}
3024
mflodmancc3d4422017-08-03 08:27:51 -07003025TEST_F(VideoStreamEncoderTest,
3026 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003027 video_stream_encoder_->OnBitrateUpdated(
3028 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003029 const int kWidth = 640;
3030 const int kHeight = 360;
3031
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003032 video_stream_encoder_->SetSource(&video_source_,
3033 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003034
3035 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3036 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003037 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003038 }
3039
mflodmancc3d4422017-08-03 08:27:51 -07003040 video_stream_encoder_->Stop();
3041 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003042 stats_proxy_.reset();
3043
3044 EXPECT_EQ(0,
3045 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3046}
3047
mflodmancc3d4422017-08-03 08:27:51 -07003048TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003049 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003050 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003051
3052 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003053 const VideoBitrateAllocation expected_bitrate =
sprang57c2fff2017-01-16 06:24:02 -08003054 DefaultVideoBitrateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003055 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3056 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003057
sprang57c2fff2017-01-16 06:24:02 -08003058 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003059 .Times(1);
Erik Språng610c7632019-03-06 15:37:33 +01003060 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003061 DataRate::bps(kLowTargetBitrateBps),
3062 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003063
sprang57c2fff2017-01-16 06:24:02 -08003064 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003065 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3066 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003067 VideoBitrateAllocation bitrate_allocation =
3068 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003069 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003070 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003071 // TODO(srte): The use of millisecs here looks like an error, but the tests
3072 // fails using seconds, this should be investigated.
3073 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003074
3075 // Not called on second frame.
3076 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3077 .Times(0);
3078 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003079 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3080 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02003081 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003082
3083 // Called after a process interval.
3084 const int64_t kProcessIntervalMs =
3085 vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
sprang57c2fff2017-01-16 06:24:02 -08003086 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3087 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003088 const int64_t start_time_ms = rtc::TimeMillis();
3089 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3090 video_source_.IncomingCapturedFrame(
3091 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3092 WaitForEncodedFrame(rtc::TimeMillis());
Sebastian Jansson40889f32019-04-17 12:11:20 +02003093 fake_clock_.AdvanceTime(TimeDelta::ms(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003094 }
3095
3096 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003097 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003098
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003100}
3101
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003102TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3103 // 2 TLs configured, temporal layers supported by encoder.
3104 const int kNumTemporalLayers = 2;
3105 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3106 fake_encoder_.SetTemporalLayersSupported(0, true);
3107
3108 // Bitrate allocated across temporal layers.
3109 const int kTl0Bps = kTargetBitrateBps *
3110 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3111 kNumTemporalLayers, /*temporal_id*/ 0);
3112 const int kTl1Bps = kTargetBitrateBps *
3113 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3114 kNumTemporalLayers, /*temporal_id*/ 1);
3115 VideoBitrateAllocation expected_bitrate;
3116 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3117 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3118
3119 VerifyAllocatedBitrate(expected_bitrate);
3120 video_stream_encoder_->Stop();
3121}
3122
3123TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3124 // 2 TLs configured, temporal layers not supported by encoder.
3125 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3126 fake_encoder_.SetTemporalLayersSupported(0, false);
3127
3128 // Temporal layers not supported by the encoder.
3129 // Total bitrate should be at ti:0.
3130 VideoBitrateAllocation expected_bitrate;
3131 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3132
3133 VerifyAllocatedBitrate(expected_bitrate);
3134 video_stream_encoder_->Stop();
3135}
3136
3137TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3138 // 2 TLs configured, temporal layers only supported for first stream.
3139 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3140 fake_encoder_.SetTemporalLayersSupported(0, true);
3141 fake_encoder_.SetTemporalLayersSupported(1, false);
3142
3143 const int kS0Bps = 150000;
3144 const int kS0Tl0Bps =
3145 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3146 /*num_layers*/ 2, /*temporal_id*/ 0);
3147 const int kS0Tl1Bps =
3148 kS0Bps * webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3149 /*num_layers*/ 2, /*temporal_id*/ 1);
3150 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3151 // Temporal layers not supported by si:1.
3152 VideoBitrateAllocation expected_bitrate;
3153 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3154 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3155 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3156
3157 VerifyAllocatedBitrate(expected_bitrate);
3158 video_stream_encoder_->Stop();
3159}
3160
Niels Möller7dc26b72017-12-06 10:27:48 +01003161TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3162 const int kFrameWidth = 1280;
3163 const int kFrameHeight = 720;
3164 const int kFramerate = 24;
3165
Erik Språng4c6ca302019-04-08 15:14:01 +02003166 video_stream_encoder_->OnBitrateUpdated(
3167 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003168 test::FrameForwarder source;
3169 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003170 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003171
3172 // Insert a single frame, triggering initial configuration.
3173 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3174 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3175
3176 EXPECT_EQ(
3177 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3178 kDefaultFramerate);
3179
3180 // Trigger reconfigure encoder (without resetting the entire instance).
3181 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003182 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003183 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3184 video_encoder_config.number_of_streams = 1;
3185 video_encoder_config.video_stream_factory =
3186 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3187 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003188 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003189 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3190
3191 // Detector should be updated with fps limit from codec config.
3192 EXPECT_EQ(
3193 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3194 kFramerate);
3195
3196 // Trigger overuse, max framerate should be reduced.
3197 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3198 stats.input_frame_rate = kFramerate;
3199 stats_proxy_->SetMockStats(stats);
3200 video_stream_encoder_->TriggerCpuOveruse();
3201 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3202 int adapted_framerate =
3203 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3204 EXPECT_LT(adapted_framerate, kFramerate);
3205
3206 // Trigger underuse, max framerate should go back to codec configured fps.
3207 // Set extra low fps, to make sure it's actually reset, not just incremented.
3208 stats = stats_proxy_->GetStats();
3209 stats.input_frame_rate = adapted_framerate / 2;
3210 stats_proxy_->SetMockStats(stats);
3211 video_stream_encoder_->TriggerCpuNormalUsage();
3212 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3213 EXPECT_EQ(
3214 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3215 kFramerate);
3216
3217 video_stream_encoder_->Stop();
3218}
3219
3220TEST_F(VideoStreamEncoderTest,
3221 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3222 const int kFrameWidth = 1280;
3223 const int kFrameHeight = 720;
3224 const int kLowFramerate = 15;
3225 const int kHighFramerate = 25;
3226
Erik Språng4c6ca302019-04-08 15:14:01 +02003227 video_stream_encoder_->OnBitrateUpdated(
3228 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003229 test::FrameForwarder source;
3230 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003231 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003232
3233 // Trigger initial configuration.
3234 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003235 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003236 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3237 video_encoder_config.number_of_streams = 1;
3238 video_encoder_config.video_stream_factory =
3239 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3240 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3241 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003242 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003243 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3244
3245 EXPECT_EQ(
3246 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3247 kLowFramerate);
3248
3249 // Trigger overuse, max framerate should be reduced.
3250 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3251 stats.input_frame_rate = kLowFramerate;
3252 stats_proxy_->SetMockStats(stats);
3253 video_stream_encoder_->TriggerCpuOveruse();
3254 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3255 int adapted_framerate =
3256 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3257 EXPECT_LT(adapted_framerate, kLowFramerate);
3258
3259 // Reconfigure the encoder with a new (higher max framerate), max fps should
3260 // still respect the adaptation.
3261 video_encoder_config.video_stream_factory =
3262 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3263 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3264 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003265 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003266 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3267
3268 EXPECT_EQ(
3269 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3270 adapted_framerate);
3271
3272 // Trigger underuse, max framerate should go back to codec configured fps.
3273 stats = stats_proxy_->GetStats();
3274 stats.input_frame_rate = adapted_framerate;
3275 stats_proxy_->SetMockStats(stats);
3276 video_stream_encoder_->TriggerCpuNormalUsage();
3277 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3278 EXPECT_EQ(
3279 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3280 kHighFramerate);
3281
3282 video_stream_encoder_->Stop();
3283}
3284
mflodmancc3d4422017-08-03 08:27:51 -07003285TEST_F(VideoStreamEncoderTest,
3286 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003287 const int kFrameWidth = 1280;
3288 const int kFrameHeight = 720;
3289 const int kFramerate = 24;
3290
Erik Språng4c6ca302019-04-08 15:14:01 +02003291 video_stream_encoder_->OnBitrateUpdated(
3292 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003293 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003294 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003295 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003296
3297 // Trigger initial configuration.
3298 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003299 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003300 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3301 video_encoder_config.number_of_streams = 1;
3302 video_encoder_config.video_stream_factory =
3303 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3304 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003305 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003306 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003307 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003308
Niels Möller7dc26b72017-12-06 10:27:48 +01003309 EXPECT_EQ(
3310 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3311 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003312
3313 // Trigger overuse, max framerate should be reduced.
3314 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3315 stats.input_frame_rate = kFramerate;
3316 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003317 video_stream_encoder_->TriggerCpuOveruse();
3318 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003319 int adapted_framerate =
3320 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003321 EXPECT_LT(adapted_framerate, kFramerate);
3322
3323 // Change degradation preference to not enable framerate scaling. Target
3324 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003325 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003326 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003327 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003328 EXPECT_EQ(
3329 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3330 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003331
mflodmancc3d4422017-08-03 08:27:51 -07003332 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003333}
3334
mflodmancc3d4422017-08-03 08:27:51 -07003335TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003336 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003337 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003338 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3339 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003340 const int kWidth = 640;
3341 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003342
asaperssonfab67072017-04-04 05:51:49 -07003343 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003344
3345 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003346 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003347
3348 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003349 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003350
sprangc5d62e22017-04-02 23:53:04 -07003351 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003352
asaperssonfab67072017-04-04 05:51:49 -07003353 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003354 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003355 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003356
3357 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003358 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003359
sprangc5d62e22017-04-02 23:53:04 -07003360 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003361
mflodmancc3d4422017-08-03 08:27:51 -07003362 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003363}
3364
mflodmancc3d4422017-08-03 08:27:51 -07003365TEST_F(VideoStreamEncoderTest,
3366 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003367 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003368 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003369 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3370 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003371 const int kWidth = 640;
3372 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003373
3374 // We expect the n initial frames to get dropped.
3375 int i;
3376 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003377 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003378 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003379 }
3380 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003381 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003382 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003383
3384 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003385 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003386
mflodmancc3d4422017-08-03 08:27:51 -07003387 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003388}
3389
mflodmancc3d4422017-08-03 08:27:51 -07003390TEST_F(VideoStreamEncoderTest,
3391 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003392 const int kWidth = 640;
3393 const int kHeight = 360;
Erik Språng610c7632019-03-06 15:37:33 +01003394 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003395 DataRate::bps(kLowTargetBitrateBps),
3396 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003397
3398 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003399 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003400 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003401
asaperssonfab67072017-04-04 05:51:49 -07003402 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003403 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003404 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003405
mflodmancc3d4422017-08-03 08:27:51 -07003406 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003407}
3408
mflodmancc3d4422017-08-03 08:27:51 -07003409TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003410 const int kWidth = 640;
3411 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003412 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003413
3414 VideoEncoderConfig video_encoder_config;
3415 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3416 // Make format different, to force recreation of encoder.
3417 video_encoder_config.video_format.parameters["foo"] = "foo";
3418 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003419 kMaxPayloadLength);
Erik Språng610c7632019-03-06 15:37:33 +01003420 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(kLowTargetBitrateBps),
Erik Språng4c6ca302019-04-08 15:14:01 +02003421 DataRate::bps(kLowTargetBitrateBps),
3422 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003423
kthelgasonb83797b2017-02-14 11:57:25 -08003424 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003425 video_stream_encoder_->SetSource(&video_source_,
3426 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003427
asaperssonfab67072017-04-04 05:51:49 -07003428 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003429 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003430 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003431
mflodmancc3d4422017-08-03 08:27:51 -07003432 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003433 fake_encoder_.SetQualityScaling(true);
3434}
3435
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003436TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBWEstimateReady) {
3437 webrtc::test::ScopedFieldTrials field_trials(
3438 "WebRTC-InitialFramedrop/Enabled/");
3439 // Reset encoder for field trials to take effect.
3440 ConfigureEncoder(video_encoder_config_.Copy());
3441 const int kTooLowBitrateForFrameSizeBps = 10000;
3442 const int kWidth = 640;
3443 const int kHeight = 360;
3444
Erik Språng4c6ca302019-04-08 15:14:01 +02003445 video_stream_encoder_->OnBitrateUpdated(
3446 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003447 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3448 // Frame should not be dropped.
3449 WaitForEncodedFrame(1);
3450
Erik Språng610c7632019-03-06 15:37:33 +01003451 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003452 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3453 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
Kári Tristan Helgason639602a2018-08-02 10:51:40 +02003454 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3455 // Expect to drop this frame, the wait should time out.
3456 ExpectDroppedFrame();
3457
3458 // Expect the sink_wants to specify a scaled frame.
3459 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3460 video_stream_encoder_->Stop();
3461}
3462
Åsa Persson139f4dc2019-08-02 09:29:58 +02003463TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3464 webrtc::test::ScopedFieldTrials field_trials(
3465 "WebRTC-Video-QualityScalerSettings/"
3466 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3467 // Reset encoder for field trials to take effect.
3468 ConfigureEncoder(video_encoder_config_.Copy());
3469 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3470 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3471 const int kWidth = 640;
3472 const int kHeight = 360;
3473
3474 video_stream_encoder_->OnBitrateUpdated(
3475 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
3476 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3477 // Frame should not be dropped.
3478 WaitForEncodedFrame(1);
3479
3480 video_stream_encoder_->OnBitrateUpdated(
3481 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
3482 DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0);
3483 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3484 // Frame should not be dropped.
3485 WaitForEncodedFrame(2);
3486
3487 video_stream_encoder_->OnBitrateUpdated(
3488 DataRate::bps(kTooLowBitrateForFrameSizeBps),
3489 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0);
3490 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3491 // Expect to drop this frame, the wait should time out.
3492 ExpectDroppedFrame();
3493
3494 // Expect the sink_wants to specify a scaled frame.
3495 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3496 video_stream_encoder_->Stop();
3497}
3498
mflodmancc3d4422017-08-03 08:27:51 -07003499TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003500 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3501 const int kTooSmallWidth = 10;
3502 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02003503 video_stream_encoder_->OnBitrateUpdated(
3504 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003505
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003506 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003507 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003508 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003509 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003510 VerifyNoLimitation(source.sink_wants());
3511 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3512
3513 // Trigger adapt down, too small frame, expect no change.
3514 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003515 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003516 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003517 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003518 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3519 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3520
mflodmancc3d4422017-08-03 08:27:51 -07003521 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003522}
3523
mflodmancc3d4422017-08-03 08:27:51 -07003524TEST_F(VideoStreamEncoderTest,
3525 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003526 const int kTooSmallWidth = 10;
3527 const int kTooSmallHeight = 10;
3528 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02003529 video_stream_encoder_->OnBitrateUpdated(
3530 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003531
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003532 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003533 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003534 video_stream_encoder_->SetSource(&source,
3535 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003536 VerifyNoLimitation(source.sink_wants());
3537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3539
3540 // Trigger adapt down, expect limited framerate.
3541 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003542 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003543 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003544 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3547 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3548
3549 // Trigger adapt down, too small frame, expect no change.
3550 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003551 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003552 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003553 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3556 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3557
mflodmancc3d4422017-08-03 08:27:51 -07003558 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003559}
3560
mflodmancc3d4422017-08-03 08:27:51 -07003561TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07003562 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02003563 video_stream_encoder_->OnBitrateUpdated(
3564 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02003565 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07003566 const int kFrameWidth = 1280;
3567 const int kFrameHeight = 720;
3568 video_source_.IncomingCapturedFrame(
3569 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003570 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07003571 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003572}
3573
sprangb1ca0732017-02-01 08:38:12 -08003574// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07003575TEST_F(VideoStreamEncoderTest,
3576 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003577 video_stream_encoder_->OnBitrateUpdated(
3578 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08003579
3580 const int kFrameWidth = 1280;
3581 const int kFrameHeight = 720;
3582 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07003583 // requested by
3584 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08003585 video_source_.set_adaptation_enabled(true);
3586
3587 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003588 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003589 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003590
3591 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003592 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08003593 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003594 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003595 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08003596
asaperssonfab67072017-04-04 05:51:49 -07003597 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003598 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08003599 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003600 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003601 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003602
mflodmancc3d4422017-08-03 08:27:51 -07003603 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08003604}
sprangfe627f32017-03-29 08:24:59 -07003605
mflodmancc3d4422017-08-03 08:27:51 -07003606TEST_F(VideoStreamEncoderTest,
3607 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07003608 const int kFrameWidth = 1280;
3609 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07003610
Erik Språng4c6ca302019-04-08 15:14:01 +02003611 video_stream_encoder_->OnBitrateUpdated(
3612 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003613 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003614 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003615 video_source_.set_adaptation_enabled(true);
3616
sprang4847ae62017-06-27 07:06:52 -07003617 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003618
3619 video_source_.IncomingCapturedFrame(
3620 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003621 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003622
3623 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07003624 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003625
3626 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07003627 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003628 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003629 video_source_.IncomingCapturedFrame(
3630 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003631 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003632 }
3633
3634 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07003635 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003636 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003637 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003638 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003639 video_source_.IncomingCapturedFrame(
3640 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003641 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003642 ++num_frames_dropped;
3643 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003644 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003645 }
3646 }
3647
sprang4847ae62017-06-27 07:06:52 -07003648 // Add some slack to account for frames dropped by the frame dropper.
3649 const int kErrorMargin = 1;
3650 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003651 kErrorMargin);
3652
3653 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07003654 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003655 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003656 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003657 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003658 video_source_.IncomingCapturedFrame(
3659 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003660 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003661 ++num_frames_dropped;
3662 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003663 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003664 }
3665 }
sprang4847ae62017-06-27 07:06:52 -07003666 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07003667 kErrorMargin);
3668
3669 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07003670 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003671 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003672 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003673 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003674 video_source_.IncomingCapturedFrame(
3675 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003676 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003677 ++num_frames_dropped;
3678 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003679 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003680 }
3681 }
sprang4847ae62017-06-27 07:06:52 -07003682 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07003683 kErrorMargin);
3684
3685 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07003686 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07003687 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003688 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003689 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003690 video_source_.IncomingCapturedFrame(
3691 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003692 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07003693 ++num_frames_dropped;
3694 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01003695 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07003696 }
3697 }
3698 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
3699
mflodmancc3d4422017-08-03 08:27:51 -07003700 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003701}
3702
mflodmancc3d4422017-08-03 08:27:51 -07003703TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07003704 const int kFramerateFps = 5;
3705 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07003706 const int kFrameWidth = 1280;
3707 const int kFrameHeight = 720;
3708
sprang4847ae62017-06-27 07:06:52 -07003709 // Reconfigure encoder with two temporal layers and screensharing, which will
3710 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02003711 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07003712
Erik Språng4c6ca302019-04-08 15:14:01 +02003713 video_stream_encoder_->OnBitrateUpdated(
3714 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003715 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003716 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003717 video_source_.set_adaptation_enabled(true);
3718
sprang4847ae62017-06-27 07:06:52 -07003719 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003720
3721 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08003722 rtc::VideoSinkWants last_wants;
3723 do {
3724 last_wants = video_source_.sink_wants();
3725
sprangc5d62e22017-04-02 23:53:04 -07003726 // Insert frames to get a new fps estimate...
3727 for (int j = 0; j < kFramerateFps; ++j) {
3728 video_source_.IncomingCapturedFrame(
3729 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08003730 if (video_source_.last_sent_width()) {
3731 sink_.WaitForEncodedFrame(timestamp_ms);
3732 }
sprangc5d62e22017-04-02 23:53:04 -07003733 timestamp_ms += kFrameIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02003734 fake_clock_.AdvanceTime(TimeDelta::ms(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07003735 }
3736 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07003737 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08003738 } while (video_source_.sink_wants().max_framerate_fps <
3739 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07003740
Jonathan Yubc771b72017-12-08 17:04:29 -08003741 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07003742
mflodmancc3d4422017-08-03 08:27:51 -07003743 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07003744}
asaperssonf7e294d2017-06-13 23:25:22 -07003745
mflodmancc3d4422017-08-03 08:27:51 -07003746TEST_F(VideoStreamEncoderTest,
3747 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003748 const int kWidth = 1280;
3749 const int kHeight = 720;
3750 const int64_t kFrameIntervalMs = 150;
3751 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003752 video_stream_encoder_->OnBitrateUpdated(
3753 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003754
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003755 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003756 AdaptingFrameForwarder source;
3757 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003758 video_stream_encoder_->SetSource(&source,
3759 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003760 timestamp_ms += kFrameIntervalMs;
3761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003762 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003763 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003764 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3765 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3766 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3767
3768 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003769 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003770 timestamp_ms += kFrameIntervalMs;
3771 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003772 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003773 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3774 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3775 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3776 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3777
3778 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003779 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003780 timestamp_ms += kFrameIntervalMs;
3781 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003782 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003783 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3784 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3785 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3786 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3787
3788 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003789 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003790 timestamp_ms += kFrameIntervalMs;
3791 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003792 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003793 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3794 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3795 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3796 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3797
3798 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003799 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003800 timestamp_ms += kFrameIntervalMs;
3801 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003802 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003803 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3804 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3805 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3806 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3807
3808 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003809 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003810 timestamp_ms += kFrameIntervalMs;
3811 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003812 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003813 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3814 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3815 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3816 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3817
3818 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003819 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003820 timestamp_ms += kFrameIntervalMs;
3821 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003822 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003823 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
3824 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3825 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3826 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3827
3828 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07003829 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003830 timestamp_ms += kFrameIntervalMs;
3831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003832 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003833 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3834 rtc::VideoSinkWants last_wants = source.sink_wants();
3835 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3836 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3837 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3838
3839 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003840 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003841 timestamp_ms += kFrameIntervalMs;
3842 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003843 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003844 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
3845 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3846 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3847 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3848
3849 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003850 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003851 timestamp_ms += kFrameIntervalMs;
3852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003853 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003854 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3855 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3856 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3857 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3858
3859 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07003860 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003861 timestamp_ms += kFrameIntervalMs;
3862 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003863 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003864 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3865 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3866 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3867 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3868
3869 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003870 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003871 timestamp_ms += kFrameIntervalMs;
3872 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003873 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003874 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
3875 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3876 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3877 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3878
3879 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003880 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003881 timestamp_ms += kFrameIntervalMs;
3882 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003883 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003884 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
3885 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3886 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3887 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3888
3889 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003890 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003891 timestamp_ms += kFrameIntervalMs;
3892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003893 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003894 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3897 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3898
3899 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003900 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003901 timestamp_ms += kFrameIntervalMs;
3902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003903 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003904 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3906 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3907 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3908
Åsa Persson30ab0152019-08-27 12:22:33 +02003909 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003910 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07003911 timestamp_ms += kFrameIntervalMs;
3912 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003913 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07003914 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003915 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003916 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3917 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3918 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3919
3920 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003921 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003922 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003923 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3924
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003926}
3927
mflodmancc3d4422017-08-03 08:27:51 -07003928TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07003929 const int kWidth = 1280;
3930 const int kHeight = 720;
3931 const int64_t kFrameIntervalMs = 150;
3932 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02003933 video_stream_encoder_->OnBitrateUpdated(
3934 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003935
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003936 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003937 AdaptingFrameForwarder source;
3938 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003939 video_stream_encoder_->SetSource(&source,
3940 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003941 timestamp_ms += kFrameIntervalMs;
3942 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003943 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003944 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003945 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3946 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3947 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3948 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3949 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3950 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3951
3952 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003953 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003954 timestamp_ms += kFrameIntervalMs;
3955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003956 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003957 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3958 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3959 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3960 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3961 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3962 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3963 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3964
3965 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003966 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07003967 timestamp_ms += kFrameIntervalMs;
3968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003969 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003970 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
3971 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3972 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3973 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3974 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3975 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3976 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3977
3978 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07003979 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003980 timestamp_ms += kFrameIntervalMs;
3981 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003982 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003983 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
3984 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3985 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3986 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3987 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3988 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3989 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3990
3991 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07003992 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07003993 timestamp_ms += kFrameIntervalMs;
3994 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003995 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003996 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
3997 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3998 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3999 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4000 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4001 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4002 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4003
4004 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004005 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004006 timestamp_ms += kFrameIntervalMs;
4007 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004008 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004009 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4010 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4011 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4012 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4013 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4014 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4015 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4016
4017 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004018 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004019 timestamp_ms += kFrameIntervalMs;
4020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004021 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004022 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004023 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4025 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4026 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4027 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4028 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4029 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4030
4031 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004032 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004033 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004034 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4035 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4036
mflodmancc3d4422017-08-03 08:27:51 -07004037 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004038}
4039
mflodmancc3d4422017-08-03 08:27:51 -07004040TEST_F(VideoStreamEncoderTest,
4041 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004042 const int kWidth = 640;
4043 const int kHeight = 360;
4044 const int kFpsLimit = 15;
4045 const int64_t kFrameIntervalMs = 150;
4046 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004047 video_stream_encoder_->OnBitrateUpdated(
4048 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004049
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004050 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004051 AdaptingFrameForwarder source;
4052 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004053 video_stream_encoder_->SetSource(&source,
4054 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004055 timestamp_ms += kFrameIntervalMs;
4056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004057 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004058 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004059 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4060 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4061 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4062 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4063 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4064 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4065
4066 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004067 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004068 timestamp_ms += kFrameIntervalMs;
4069 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004070 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004071 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4072 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4073 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4074 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4076 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4077 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4078
4079 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004080 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004081 timestamp_ms += kFrameIntervalMs;
4082 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004083 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004084 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4085 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4086 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4087 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4088 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4089 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4090 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4091
4092 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004093 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004094 timestamp_ms += kFrameIntervalMs;
4095 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004096 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004097 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4098 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4099 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4100 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4101 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4102 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4103 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4104
4105 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004106 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004107 timestamp_ms += kFrameIntervalMs;
4108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004109 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004110 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004111 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4112 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4113 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4114 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4115 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4116 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4117
4118 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004119 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004120 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004121 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4122 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4123
mflodmancc3d4422017-08-03 08:27:51 -07004124 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004125}
4126
mflodmancc3d4422017-08-03 08:27:51 -07004127TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004128 const int kFrameWidth = 1920;
4129 const int kFrameHeight = 1080;
4130 // 3/4 of 1920.
4131 const int kAdaptedFrameWidth = 1440;
4132 // 3/4 of 1080 rounded down to multiple of 4.
4133 const int kAdaptedFrameHeight = 808;
4134 const int kFramerate = 24;
4135
Erik Språng4c6ca302019-04-08 15:14:01 +02004136 video_stream_encoder_->OnBitrateUpdated(
4137 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004138 // Trigger reconfigure encoder (without resetting the entire instance).
4139 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004140 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004141 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4142 video_encoder_config.number_of_streams = 1;
4143 video_encoder_config.video_stream_factory =
4144 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004145 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004146 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004147 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004148
4149 video_source_.set_adaptation_enabled(true);
4150
4151 video_source_.IncomingCapturedFrame(
4152 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004153 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004154
4155 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004156 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004157 video_source_.IncomingCapturedFrame(
4158 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004159 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004160
mflodmancc3d4422017-08-03 08:27:51 -07004161 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004162}
4163
mflodmancc3d4422017-08-03 08:27:51 -07004164TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004165 const int kFrameWidth = 1280;
4166 const int kFrameHeight = 720;
4167 const int kLowFps = 2;
4168 const int kHighFps = 30;
4169
Erik Språng4c6ca302019-04-08 15:14:01 +02004170 video_stream_encoder_->OnBitrateUpdated(
4171 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004172
4173 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4174 max_framerate_ = kLowFps;
4175
4176 // Insert 2 seconds of 2fps video.
4177 for (int i = 0; i < kLowFps * 2; ++i) {
4178 video_source_.IncomingCapturedFrame(
4179 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4180 WaitForEncodedFrame(timestamp_ms);
4181 timestamp_ms += 1000 / kLowFps;
4182 }
4183
4184 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004185 video_stream_encoder_->OnBitrateUpdated(
4186 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004187 video_source_.IncomingCapturedFrame(
4188 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4189 WaitForEncodedFrame(timestamp_ms);
4190 timestamp_ms += 1000 / kLowFps;
4191
4192 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4193
4194 // Insert 30fps frames for just a little more than the forced update period.
4195 const int kVcmTimerIntervalFrames =
4196 (vcm::VCMProcessTimer::kDefaultProcessIntervalMs * kHighFps) / 1000;
4197 const int kFrameIntervalMs = 1000 / kHighFps;
4198 max_framerate_ = kHighFps;
4199 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4200 video_source_.IncomingCapturedFrame(
4201 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4202 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4203 // be dropped if the encoder hans't been updated with the new higher target
4204 // framerate yet, causing it to overshoot the target bitrate and then
4205 // suffering the wrath of the media optimizer.
4206 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4207 timestamp_ms += kFrameIntervalMs;
4208 }
4209
4210 // Don expect correct measurement just yet, but it should be higher than
4211 // before.
4212 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4213
mflodmancc3d4422017-08-03 08:27:51 -07004214 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004215}
4216
mflodmancc3d4422017-08-03 08:27:51 -07004217TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004218 const int kFrameWidth = 1280;
4219 const int kFrameHeight = 720;
4220 const int kTargetBitrateBps = 1000000;
4221
4222 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004223 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004224 video_stream_encoder_->OnBitrateUpdated(
4225 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004226 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004227
4228 // Insert a first video frame, causes another bitrate update.
4229 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4230 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4231 video_source_.IncomingCapturedFrame(
4232 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4233 WaitForEncodedFrame(timestamp_ms);
4234
4235 // Next, simulate video suspension due to pacer queue overrun.
Erik Språng4c6ca302019-04-08 15:14:01 +02004236 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0), 0,
Erik Språng610c7632019-03-06 15:37:33 +01004237 1);
sprang4847ae62017-06-27 07:06:52 -07004238
4239 // Skip ahead until a new periodic parameter update should have occured.
4240 timestamp_ms += vcm::VCMProcessTimer::kDefaultProcessIntervalMs;
Sebastian Jansson40889f32019-04-17 12:11:20 +02004241 fake_clock_.AdvanceTime(
4242 TimeDelta::ms(vcm::VCMProcessTimer::kDefaultProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004243
4244 // Bitrate observer should not be called.
4245 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4246 video_source_.IncomingCapturedFrame(
4247 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4248 ExpectDroppedFrame();
4249
mflodmancc3d4422017-08-03 08:27:51 -07004250 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004251}
ilnik6b826ef2017-06-16 06:53:48 -07004252
Niels Möller4db138e2018-04-19 09:04:13 +02004253TEST_F(VideoStreamEncoderTest,
4254 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4255 const int kFrameWidth = 1280;
4256 const int kFrameHeight = 720;
4257 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004258 video_stream_encoder_->OnBitrateUpdated(
4259 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004260 video_source_.IncomingCapturedFrame(
4261 CreateFrame(1, kFrameWidth, kFrameHeight));
4262 WaitForEncodedFrame(1);
4263 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4264 .low_encode_usage_threshold_percent,
4265 default_options.low_encode_usage_threshold_percent);
4266 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4267 .high_encode_usage_threshold_percent,
4268 default_options.high_encode_usage_threshold_percent);
4269 video_stream_encoder_->Stop();
4270}
4271
4272TEST_F(VideoStreamEncoderTest,
4273 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4274 const int kFrameWidth = 1280;
4275 const int kFrameHeight = 720;
4276 CpuOveruseOptions hardware_options;
4277 hardware_options.low_encode_usage_threshold_percent = 150;
4278 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004279 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004280
Erik Språng4c6ca302019-04-08 15:14:01 +02004281 video_stream_encoder_->OnBitrateUpdated(
4282 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004283 video_source_.IncomingCapturedFrame(
4284 CreateFrame(1, kFrameWidth, kFrameHeight));
4285 WaitForEncodedFrame(1);
4286 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4287 .low_encode_usage_threshold_percent,
4288 hardware_options.low_encode_usage_threshold_percent);
4289 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4290 .high_encode_usage_threshold_percent,
4291 hardware_options.high_encode_usage_threshold_percent);
4292 video_stream_encoder_->Stop();
4293}
4294
Niels Möller6bb5ab92019-01-11 11:11:10 +01004295TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4296 const int kFrameWidth = 320;
4297 const int kFrameHeight = 240;
4298 const int kFps = 30;
4299 const int kTargetBitrateBps = 120000;
4300 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4301
Erik Språng4c6ca302019-04-08 15:14:01 +02004302 video_stream_encoder_->OnBitrateUpdated(
4303 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004304
4305 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4306 max_framerate_ = kFps;
4307
4308 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4309 fake_encoder_.SimulateOvershoot(1.0);
4310 int num_dropped = 0;
4311 for (int i = 0; i < kNumFramesInRun; ++i) {
4312 video_source_.IncomingCapturedFrame(
4313 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4314 // Wait up to two frame durations for a frame to arrive.
4315 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4316 ++num_dropped;
4317 }
4318 timestamp_ms += 1000 / kFps;
4319 }
4320
Erik Språnga8d48ab2019-02-08 14:17:40 +01004321 // Framerate should be measured to be near the expected target rate.
4322 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4323
4324 // Frame drops should be within 5% of expected 0%.
4325 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004326
4327 // Make encoder produce frames at double the expected bitrate during 3 seconds
4328 // of video, verify number of drops. Rate needs to be slightly changed in
4329 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004330 double overshoot_factor = 2.0;
4331 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4332 // With bitrate adjuster, when need to overshoot even more to trigger
4333 // frame dropping.
4334 overshoot_factor *= 2;
4335 }
4336 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004337 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004338 DataRate::bps(kTargetBitrateBps + 1000),
4339 DataRate::bps(kTargetBitrateBps + 1000), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004340 num_dropped = 0;
4341 for (int i = 0; i < kNumFramesInRun; ++i) {
4342 video_source_.IncomingCapturedFrame(
4343 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4344 // Wait up to two frame durations for a frame to arrive.
4345 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4346 ++num_dropped;
4347 }
4348 timestamp_ms += 1000 / kFps;
4349 }
4350
Erik Språng4c6ca302019-04-08 15:14:01 +02004351 video_stream_encoder_->OnBitrateUpdated(
4352 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004353
4354 // Target framerate should be still be near the expected target, despite
4355 // the frame drops.
4356 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4357
4358 // Frame drops should be within 5% of expected 50%.
4359 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004360
4361 video_stream_encoder_->Stop();
4362}
4363
4364TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4365 const int kFrameWidth = 320;
4366 const int kFrameHeight = 240;
4367 const int kActualInputFps = 24;
4368 const int kTargetBitrateBps = 120000;
4369
4370 ASSERT_GT(max_framerate_, kActualInputFps);
4371
4372 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4373 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004374 video_stream_encoder_->OnBitrateUpdated(
4375 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004376
4377 // Insert 3 seconds of video, with an input fps lower than configured max.
4378 for (int i = 0; i < kActualInputFps * 3; ++i) {
4379 video_source_.IncomingCapturedFrame(
4380 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4381 // Wait up to two frame durations for a frame to arrive.
4382 WaitForEncodedFrame(timestamp_ms);
4383 timestamp_ms += 1000 / kActualInputFps;
4384 }
4385
4386 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4387
4388 video_stream_encoder_->Stop();
4389}
4390
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004391TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4392 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004393 video_stream_encoder_->OnBitrateUpdated(
4394 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004395
4396 fake_encoder_.BlockNextEncode();
4397 video_source_.IncomingCapturedFrame(
4398 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4399 WaitForEncodedFrame(1);
4400 // On the very first frame full update should be forced.
4401 rect = fake_encoder_.GetLastUpdateRect();
4402 EXPECT_EQ(rect.offset_x, 0);
4403 EXPECT_EQ(rect.offset_y, 0);
4404 EXPECT_EQ(rect.height, codec_height_);
4405 EXPECT_EQ(rect.width, codec_width_);
4406 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4407 // call to ContinueEncode.
4408 video_source_.IncomingCapturedFrame(
4409 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4410 ExpectDroppedFrame();
4411 video_source_.IncomingCapturedFrame(
4412 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4413 ExpectDroppedFrame();
4414 fake_encoder_.ContinueEncode();
4415 WaitForEncodedFrame(3);
4416 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4417 rect = fake_encoder_.GetLastUpdateRect();
4418 EXPECT_EQ(rect.offset_x, 1);
4419 EXPECT_EQ(rect.offset_y, 0);
4420 EXPECT_EQ(rect.width, 10);
4421 EXPECT_EQ(rect.height, 1);
4422
4423 video_source_.IncomingCapturedFrame(
4424 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4425 WaitForEncodedFrame(4);
4426 // Previous frame was encoded, so no accumulation should happen.
4427 rect = fake_encoder_.GetLastUpdateRect();
4428 EXPECT_EQ(rect.offset_x, 0);
4429 EXPECT_EQ(rect.offset_y, 0);
4430 EXPECT_EQ(rect.width, 1);
4431 EXPECT_EQ(rect.height, 1);
4432
4433 video_stream_encoder_->Stop();
4434}
4435
Erik Språngd7329ca2019-02-21 21:19:53 +01004436TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004437 video_stream_encoder_->OnBitrateUpdated(
4438 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004439
4440 // First frame is always keyframe.
4441 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4442 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004443 EXPECT_THAT(
4444 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004445 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004446
4447 // Insert delta frame.
4448 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4449 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01004450 EXPECT_THAT(
4451 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004452 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004453
4454 // Request next frame be a key-frame.
4455 video_stream_encoder_->SendKeyFrame();
4456 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4457 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01004458 EXPECT_THAT(
4459 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004460 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004461
4462 video_stream_encoder_->Stop();
4463}
4464
4465TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4466 // Setup simulcast with three streams.
4467 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01004468 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004469 DataRate::bps(kSimulcastTargetBitrateBps),
4470 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004471 // Wait for all three layers before triggering event.
4472 sink_.SetNumExpectedLayers(3);
4473
4474 // First frame is always keyframe.
4475 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4476 WaitForEncodedFrame(1);
4477 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004478 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4479 VideoFrameType::kVideoFrameKey,
4480 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004481
4482 // Insert delta frame.
4483 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4484 WaitForEncodedFrame(2);
4485 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004486 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4487 VideoFrameType::kVideoFrameDelta,
4488 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004489
4490 // Request next frame be a key-frame.
4491 // Only first stream is configured to produce key-frame.
4492 video_stream_encoder_->SendKeyFrame();
4493 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4494 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02004495
4496 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4497 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01004498 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004499 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02004500 VideoFrameType::kVideoFrameKey,
4501 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004502
4503 video_stream_encoder_->Stop();
4504}
4505
4506TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
4507 // Configure internal source factory and setup test again.
4508 encoder_factory_.SetHasInternalSource(true);
4509 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004510 video_stream_encoder_->OnBitrateUpdated(
4511 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004512
4513 // Call encoder directly, simulating internal source where encoded frame
4514 // callback in VideoStreamEncoder is called despite no OnFrame().
4515 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
4516 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004517 EXPECT_THAT(
4518 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004519 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004520
Niels Möller8f7ce222019-03-21 15:43:58 +01004521 const std::vector<VideoFrameType> kDeltaFrame = {
4522 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01004523 // Need to set timestamp manually since manually for injected frame.
4524 VideoFrame frame = CreateFrame(101, nullptr);
4525 frame.set_timestamp(101);
4526 fake_encoder_.InjectFrame(frame, false);
4527 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004528 EXPECT_THAT(
4529 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004530 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004531
4532 // Request key-frame. The forces a dummy frame down into the encoder.
4533 fake_encoder_.ExpectNullFrame();
4534 video_stream_encoder_->SendKeyFrame();
4535 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004536 EXPECT_THAT(
4537 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004538 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004539
4540 video_stream_encoder_->Stop();
4541}
Erik Språngb7cb7b52019-02-26 15:52:33 +01004542
4543TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
4544 // Configure internal source factory and setup test again.
4545 encoder_factory_.SetHasInternalSource(true);
4546 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004547 video_stream_encoder_->OnBitrateUpdated(
4548 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01004549
4550 int64_t timestamp = 1;
4551 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02004552 image.SetEncodedData(
4553 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01004554 image.capture_time_ms_ = ++timestamp;
4555 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
4556 const int64_t kEncodeFinishDelayMs = 10;
4557 image.timing_.encode_start_ms = timestamp;
4558 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
4559 fake_encoder_.InjectEncodedImage(image);
4560 // Wait for frame without incrementing clock.
4561 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4562 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
4563 // capture timestamp should be kEncodeFinishDelayMs in the past.
4564 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
4565 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
4566 kEncodeFinishDelayMs);
4567
4568 video_stream_encoder_->Stop();
4569}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02004570
4571TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
4572 // Configure internal source factory and setup test again.
4573 encoder_factory_.SetHasInternalSource(true);
4574 ResetEncoder("H264", 1, 1, 1, false);
4575
4576 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
4577 image._frameType = VideoFrameType::kVideoFrameKey;
4578
4579 CodecSpecificInfo codec_specific_info;
4580 codec_specific_info.codecType = kVideoCodecH264;
4581
4582 RTPFragmentationHeader fragmentation;
4583 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4584 fragmentation.fragmentationOffset[0] = 4;
4585 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
4586
4587 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4588 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4589
4590 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4591 testing::ElementsAreArray(optimal_sps));
4592 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4593 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4594 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4595 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4596
4597 video_stream_encoder_->Stop();
4598}
4599
4600TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
4601 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
4602 0x00, 0x00, 0x03, 0x03, 0xF4,
4603 0x05, 0x03, 0xC7, 0xC0};
4604
4605 // Configure internal source factory and setup test again.
4606 encoder_factory_.SetHasInternalSource(true);
4607 ResetEncoder("H264", 1, 1, 1, false);
4608
4609 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
4610 image._frameType = VideoFrameType::kVideoFrameKey;
4611
4612 CodecSpecificInfo codec_specific_info;
4613 codec_specific_info.codecType = kVideoCodecH264;
4614
4615 RTPFragmentationHeader fragmentation;
4616 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4617 fragmentation.fragmentationOffset[0] = 4;
4618 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
4619
4620 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4621 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4622
4623 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4624 testing::ElementsAreArray(optimal_sps));
4625 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4626 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4627 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4628 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4629
4630 video_stream_encoder_->Stop();
4631}
4632
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02004633TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
4634 const int kFrameWidth = 1280;
4635 const int kFrameHeight = 720;
4636 const int kTargetBitrateBps = 300000; // To low for HD resolution.
4637
4638 video_stream_encoder_->OnBitrateUpdated(
4639 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps), 0, 0);
4640 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4641
4642 // Insert a first video frame. It should be dropped because of downscale in
4643 // resolution.
4644 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4645 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4646 frame.set_rotation(kVideoRotation_270);
4647 video_source_.IncomingCapturedFrame(frame);
4648
4649 ExpectDroppedFrame();
4650
4651 // Second frame is downscaled.
4652 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4653 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4654 frame.set_rotation(kVideoRotation_90);
4655 video_source_.IncomingCapturedFrame(frame);
4656
4657 WaitForEncodedFrame(timestamp_ms);
4658 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
4659
4660 // Insert another frame, also downscaled.
4661 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4662 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
4663 frame.set_rotation(kVideoRotation_180);
4664 video_source_.IncomingCapturedFrame(frame);
4665
4666 WaitForEncodedFrame(timestamp_ms);
4667 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
4668
4669 video_stream_encoder_->Stop();
4670}
4671
Erik Språng5056af02019-09-02 15:53:11 +02004672TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
4673 const int kFrameWidth = 320;
4674 const int kFrameHeight = 180;
4675
4676 // Initial rate.
4677 video_stream_encoder_->OnBitrateUpdated(
4678 /*target_bitrate=*/DataRate::kbps(300),
4679 /*link_allocation=*/DataRate::kbps(300),
4680 /*fraction_lost=*/0,
4681 /*rtt_ms=*/0);
4682
4683 // Insert a first video frame so that encoder gets configured.
4684 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4685 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
4686 frame.set_rotation(kVideoRotation_270);
4687 video_source_.IncomingCapturedFrame(frame);
4688 WaitForEncodedFrame(timestamp_ms);
4689
4690 // Set a target rate below the minimum allowed by the codec settings.
4691 VideoCodec codec_config = fake_encoder_.codec_config();
4692 DataRate min_rate = DataRate::kbps(codec_config.minBitrate);
4693 DataRate target_rate = min_rate - DataRate::kbps(1);
4694 video_stream_encoder_->OnBitrateUpdated(
4695 /*target_bitrate=*/target_rate,
4696 /*link_allocation=*/target_rate,
4697 /*fraction_lost=*/0,
4698 /*rtt_ms=*/0);
4699 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4700
4701 // Target bitrate and bandwidth allocation should both be capped at min_rate.
4702 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
4703 ASSERT_TRUE(rate_settings.has_value());
4704 DataRate allocation_sum = DataRate::bps(rate_settings->bitrate.get_sum_bps());
4705 EXPECT_EQ(min_rate, allocation_sum);
4706 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
4707
4708 video_stream_encoder_->Stop();
4709}
4710
perkj26091b12016-09-01 01:17:40 -07004711} // namespace webrtc