blob: 44a14195f4ae8543462271f7eddfe76a76976142 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010024#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020025#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020026#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010027#include "api/video_codecs/vp8_temporal_layers_factory.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020028#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070029#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080030#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020031#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010032#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010036#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020037#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "system_wrappers/include/sleep.h"
39#include "test/encoder_settings.h"
40#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020041#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010042#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020043#include "test/gmock.h"
44#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020045#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070047
48namespace webrtc {
49
sprangb1ca0732017-02-01 08:38:12 -080050using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080051using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020052using ::testing::AllOf;
53using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010054using ::testing::Matcher;
55using ::testing::NiceMock;
56using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020057using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080058
perkj803d97f2016-11-01 11:45:46 -070059namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020060const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010061const int kQpLow = 1;
62const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020063const int kMinFramerateFps = 2;
64const int kMinBalancedFramerateFps = 7;
65const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080066const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010067const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020068const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010069const uint32_t kSimulcastTargetBitrateBps = 3150000;
70const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080071const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070072const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020073const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020074const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020075const VideoEncoder::ResolutionBitrateLimits
76 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
77const VideoEncoder::ResolutionBitrateLimits
78 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080079
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020080uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
81 0x00, 0x00, 0x03, 0x03, 0xF4,
82 0x05, 0x03, 0xC7, 0xE0, 0x1B,
83 0x41, 0x10, 0x8D, 0x00};
84
perkj803d97f2016-11-01 11:45:46 -070085class TestBuffer : public webrtc::I420Buffer {
86 public:
87 TestBuffer(rtc::Event* event, int width, int height)
88 : I420Buffer(width, height), event_(event) {}
89
90 private:
91 friend class rtc::RefCountedObject<TestBuffer>;
92 ~TestBuffer() override {
93 if (event_)
94 event_->Set();
95 }
96 rtc::Event* const event_;
97};
98
Noah Richards51db4212019-06-12 06:59:12 -070099// A fake native buffer that can't be converted to I420.
100class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
101 public:
102 FakeNativeBuffer(rtc::Event* event, int width, int height)
103 : event_(event), width_(width), height_(height) {}
104 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
105 int width() const override { return width_; }
106 int height() const override { return height_; }
107 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
108 return nullptr;
109 }
110
111 private:
112 friend class rtc::RefCountedObject<FakeNativeBuffer>;
113 ~FakeNativeBuffer() override {
114 if (event_)
115 event_->Set();
116 }
117 rtc::Event* const event_;
118 const int width_;
119 const int height_;
120};
121
Niels Möller7dc26b72017-12-06 10:27:48 +0100122class CpuOveruseDetectorProxy : public OveruseFrameDetector {
123 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200124 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
125 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100126 last_target_framerate_fps_(-1) {}
127 virtual ~CpuOveruseDetectorProxy() {}
128
129 void OnTargetFramerateUpdated(int framerate_fps) override {
130 rtc::CritScope cs(&lock_);
131 last_target_framerate_fps_ = framerate_fps;
132 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
133 }
134
135 int GetLastTargetFramerate() {
136 rtc::CritScope cs(&lock_);
137 return last_target_framerate_fps_;
138 }
139
Niels Möller4db138e2018-04-19 09:04:13 +0200140 CpuOveruseOptions GetOptions() { return options_; }
141
Niels Möller7dc26b72017-12-06 10:27:48 +0100142 private:
143 rtc::CriticalSection lock_;
144 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
145};
146
mflodmancc3d4422017-08-03 08:27:51 -0700147class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700148 public:
Niels Möller213618e2018-07-24 09:29:58 +0200149 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200150 const VideoStreamEncoderSettings& settings,
151 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100152 : VideoStreamEncoder(Clock::GetRealTimeClock(),
153 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200154 stats_proxy,
155 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200156 std::unique_ptr<OveruseFrameDetector>(
157 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100158 new CpuOveruseDetectorProxy(stats_proxy)),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200159 task_queue_factory) {}
perkj803d97f2016-11-01 11:45:46 -0700160
Henrik Boströmb08882b2020-01-07 10:11:17 +0100161 void PostTaskAndWait(bool down,
162 AdaptationObserverInterface::AdaptReason reason) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200163 PostTaskAndWait(down, reason, /*expected_results=*/true);
164 }
165
Henrik Boströmb08882b2020-01-07 10:11:17 +0100166 void PostTaskAndWait(bool down,
167 AdaptationObserverInterface::AdaptReason reason,
168 bool expected_results) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100169 rtc::Event event;
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200170 encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
Åsa Perssonf5e5d252019-08-16 17:24:59 +0200171 if (down)
Henrik Boström7875c992020-02-06 10:35:00 +0100172 EXPECT_EQ(expected_results, OnResourceOveruseForTesting(reason));
Åsa Perssonf5e5d252019-08-16 17:24:59 +0200173 else
Henrik Boström7875c992020-02-06 10:35:00 +0100174 OnResourceUnderuseForTesting(reason);
perkj803d97f2016-11-01 11:45:46 -0700175 event.Set();
176 });
perkj070ba852017-02-16 15:46:27 -0800177 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700178 }
179
kthelgason2fc52542017-03-03 00:24:41 -0800180 // This is used as a synchronisation mechanism, to make sure that the
181 // encoder queue is not blocked before we start sending it frames.
182 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100183 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200184 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800185 ASSERT_TRUE(event.Wait(5000));
186 }
187
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200188 void TriggerCpuOveruse() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100189 PostTaskAndWait(/*down=*/true,
190 AdaptationObserverInterface::AdaptReason::kCpu);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200191 }
kthelgason876222f2016-11-29 01:44:11 -0800192
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200193 void TriggerCpuNormalUsage() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100194 PostTaskAndWait(/*down=*/false,
195 AdaptationObserverInterface::AdaptReason::kCpu);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200196 }
kthelgason876222f2016-11-29 01:44:11 -0800197
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200198 void TriggerQualityLow() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100199 PostTaskAndWait(/*down=*/true,
200 AdaptationObserverInterface::AdaptReason::kQuality);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200201 }
kthelgason876222f2016-11-29 01:44:11 -0800202
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200203 void TriggerQualityLowExpectFalse() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100204 PostTaskAndWait(/*down=*/true,
205 AdaptationObserverInterface::AdaptReason::kQuality,
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200206 /*expected_results=*/false);
207 }
208
209 void TriggerQualityHigh() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100210 PostTaskAndWait(/*down=*/false,
211 AdaptationObserverInterface::AdaptReason::kQuality);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200212 }
sprangfda496a2017-06-15 04:21:07 -0700213
Niels Möller7dc26b72017-12-06 10:27:48 +0100214 CpuOveruseDetectorProxy* overuse_detector_proxy_;
perkj803d97f2016-11-01 11:45:46 -0700215};
216
asapersson5f7226f2016-11-25 04:37:00 -0800217class VideoStreamFactory
218 : public VideoEncoderConfig::VideoStreamFactoryInterface {
219 public:
sprangfda496a2017-06-15 04:21:07 -0700220 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
221 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800222 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700223 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800224 }
225
226 private:
227 std::vector<VideoStream> CreateEncoderStreams(
228 int width,
229 int height,
230 const VideoEncoderConfig& encoder_config) override {
231 std::vector<VideoStream> streams =
232 test::CreateVideoStreams(width, height, encoder_config);
233 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100234 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700235 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800236 }
237 return streams;
238 }
sprangfda496a2017-06-15 04:21:07 -0700239
asapersson5f7226f2016-11-25 04:37:00 -0800240 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700241 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800242};
243
Noah Richards51db4212019-06-12 06:59:12 -0700244// Simulates simulcast behavior and makes highest stream resolutions divisible
245// by 4.
246class CroppingVideoStreamFactory
247 : public VideoEncoderConfig::VideoStreamFactoryInterface {
248 public:
249 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
250 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
251 EXPECT_GT(num_temporal_layers, 0u);
252 EXPECT_GT(framerate, 0);
253 }
254
255 private:
256 std::vector<VideoStream> CreateEncoderStreams(
257 int width,
258 int height,
259 const VideoEncoderConfig& encoder_config) override {
260 std::vector<VideoStream> streams = test::CreateVideoStreams(
261 width - width % 4, height - height % 4, encoder_config);
262 for (VideoStream& stream : streams) {
263 stream.num_temporal_layers = num_temporal_layers_;
264 stream.max_framerate = framerate_;
265 }
266 return streams;
267 }
268
269 const size_t num_temporal_layers_;
270 const int framerate_;
271};
272
sprangb1ca0732017-02-01 08:38:12 -0800273class AdaptingFrameForwarder : public test::FrameForwarder {
274 public:
275 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700276 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800277
278 void set_adaptation_enabled(bool enabled) {
279 rtc::CritScope cs(&crit_);
280 adaptation_enabled_ = enabled;
281 }
282
asaperssonfab67072017-04-04 05:51:49 -0700283 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800284 rtc::CritScope cs(&crit_);
285 return adaptation_enabled_;
286 }
287
asapersson09f05612017-05-15 23:40:18 -0700288 rtc::VideoSinkWants last_wants() const {
289 rtc::CritScope cs(&crit_);
290 return last_wants_;
291 }
292
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200293 absl::optional<int> last_sent_width() const { return last_width_; }
294 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800295
sprangb1ca0732017-02-01 08:38:12 -0800296 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
297 int cropped_width = 0;
298 int cropped_height = 0;
299 int out_width = 0;
300 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700301 if (adaption_enabled()) {
302 if (adapter_.AdaptFrameResolution(
303 video_frame.width(), video_frame.height(),
304 video_frame.timestamp_us() * 1000, &cropped_width,
305 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100306 VideoFrame adapted_frame =
307 VideoFrame::Builder()
308 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
309 nullptr, out_width, out_height))
310 .set_timestamp_rtp(99)
311 .set_timestamp_ms(99)
312 .set_rotation(kVideoRotation_0)
313 .build();
sprangc5d62e22017-04-02 23:53:04 -0700314 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100315 if (video_frame.has_update_rect()) {
316 adapted_frame.set_update_rect(
317 video_frame.update_rect().ScaleWithFrame(
318 video_frame.width(), video_frame.height(), 0, 0,
319 video_frame.width(), video_frame.height(), out_width,
320 out_height));
321 }
sprangc5d62e22017-04-02 23:53:04 -0700322 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800323 last_width_.emplace(adapted_frame.width());
324 last_height_.emplace(adapted_frame.height());
325 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200326 last_width_ = absl::nullopt;
327 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700328 }
sprangb1ca0732017-02-01 08:38:12 -0800329 } else {
330 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800331 last_width_.emplace(video_frame.width());
332 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800333 }
334 }
335
336 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
337 const rtc::VideoSinkWants& wants) override {
338 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700339 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100340 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800341 test::FrameForwarder::AddOrUpdateSink(sink, wants);
342 }
sprangb1ca0732017-02-01 08:38:12 -0800343 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700344 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
345 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200346 absl::optional<int> last_width_;
347 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800348};
sprangc5d62e22017-04-02 23:53:04 -0700349
Niels Möller213618e2018-07-24 09:29:58 +0200350// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700351class MockableSendStatisticsProxy : public SendStatisticsProxy {
352 public:
353 MockableSendStatisticsProxy(Clock* clock,
354 const VideoSendStream::Config& config,
355 VideoEncoderConfig::ContentType content_type)
356 : SendStatisticsProxy(clock, config, content_type) {}
357
358 VideoSendStream::Stats GetStats() override {
359 rtc::CritScope cs(&lock_);
360 if (mock_stats_)
361 return *mock_stats_;
362 return SendStatisticsProxy::GetStats();
363 }
364
Niels Möller213618e2018-07-24 09:29:58 +0200365 int GetInputFrameRate() const override {
366 rtc::CritScope cs(&lock_);
367 if (mock_stats_)
368 return mock_stats_->input_frame_rate;
369 return SendStatisticsProxy::GetInputFrameRate();
370 }
sprangc5d62e22017-04-02 23:53:04 -0700371 void SetMockStats(const VideoSendStream::Stats& stats) {
372 rtc::CritScope cs(&lock_);
373 mock_stats_.emplace(stats);
374 }
375
376 void ResetMockStats() {
377 rtc::CritScope cs(&lock_);
378 mock_stats_.reset();
379 }
380
381 private:
382 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200383 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700384};
385
sprang4847ae62017-06-27 07:06:52 -0700386class MockBitrateObserver : public VideoBitrateAllocationObserver {
387 public:
Erik Språng566124a2018-04-23 12:32:22 +0200388 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700389};
390
philipel9b058032020-02-10 11:30:00 +0100391class MockEncoderSelector
392 : public VideoEncoderFactory::EncoderSelectorInterface {
393 public:
394 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
395 MOCK_METHOD1(OnEncodingBitrate,
396 absl::optional<SdpVideoFormat>(const DataRate& rate));
397 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
398};
399
perkj803d97f2016-11-01 11:45:46 -0700400} // namespace
401
mflodmancc3d4422017-08-03 08:27:51 -0700402class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700403 public:
404 static const int kDefaultTimeoutMs = 30 * 1000;
405
mflodmancc3d4422017-08-03 08:27:51 -0700406 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700407 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700408 codec_width_(320),
409 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200410 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200411 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700412 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200413 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700414 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700415 Clock::GetRealTimeClock(),
416 video_send_config_,
417 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700418 sink_(&fake_encoder_) {}
419
420 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700421 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700422 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200423 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800424 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200425 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200426 video_send_config_.rtp.payload_name = "FAKE";
427 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700428
Per512ecb32016-09-23 15:52:06 +0200429 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200430 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700431 video_encoder_config.video_stream_factory =
432 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100433 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700434
435 // Framerate limit is specified by the VideoStreamFactory.
436 std::vector<VideoStream> streams =
437 video_encoder_config.video_stream_factory->CreateEncoderStreams(
438 codec_width_, codec_height_, video_encoder_config);
439 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100440 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700441
Niels Möllerf1338562018-04-26 09:51:47 +0200442 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800443 }
444
Niels Möllerf1338562018-04-26 09:51:47 +0200445 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700446 if (video_stream_encoder_)
447 video_stream_encoder_->Stop();
448 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200449 stats_proxy_.get(), video_send_config_.encoder_settings,
450 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700451 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
452 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700453 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700454 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
455 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200456 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700457 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800458 }
459
460 void ResetEncoder(const std::string& payload_name,
461 size_t num_streams,
462 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700463 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700464 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200465 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800466
467 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200468 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800469 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100470 video_encoder_config.max_bitrate_bps =
471 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800472 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700473 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
474 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700475 video_encoder_config.content_type =
476 screenshare ? VideoEncoderConfig::ContentType::kScreen
477 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700478 if (payload_name == "VP9") {
479 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
480 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
481 video_encoder_config.encoder_specific_settings =
482 new rtc::RefCountedObject<
483 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
484 }
Niels Möllerf1338562018-04-26 09:51:47 +0200485 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700486 }
487
sprang57c2fff2017-01-16 06:24:02 -0800488 VideoFrame CreateFrame(int64_t ntp_time_ms,
489 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100490 VideoFrame frame =
491 VideoFrame::Builder()
492 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
493 destruction_event, codec_width_, codec_height_))
494 .set_timestamp_rtp(99)
495 .set_timestamp_ms(99)
496 .set_rotation(kVideoRotation_0)
497 .build();
sprang57c2fff2017-01-16 06:24:02 -0800498 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700499 return frame;
500 }
501
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100502 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
503 rtc::Event* destruction_event,
504 int offset_x) const {
505 VideoFrame frame =
506 VideoFrame::Builder()
507 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
508 destruction_event, codec_width_, codec_height_))
509 .set_timestamp_rtp(99)
510 .set_timestamp_ms(99)
511 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100512 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100513 .build();
514 frame.set_ntp_time_ms(ntp_time_ms);
515 return frame;
516 }
517
sprang57c2fff2017-01-16 06:24:02 -0800518 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100519 VideoFrame frame =
520 VideoFrame::Builder()
521 .set_video_frame_buffer(
522 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
523 .set_timestamp_rtp(99)
524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
sprang57c2fff2017-01-16 06:24:02 -0800527 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700528 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700529 return frame;
530 }
531
Noah Richards51db4212019-06-12 06:59:12 -0700532 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
533 rtc::Event* destruction_event,
534 int width,
535 int height) const {
536 VideoFrame frame =
537 VideoFrame::Builder()
538 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
539 destruction_event, width, height))
540 .set_timestamp_rtp(99)
541 .set_timestamp_ms(99)
542 .set_rotation(kVideoRotation_0)
543 .build();
544 frame.set_ntp_time_ms(ntp_time_ms);
545 return frame;
546 }
547
548 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
549 rtc::Event* destruction_event) const {
550 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
551 codec_height_);
552 }
553
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100554 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
555 MockBitrateObserver bitrate_observer;
556 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
557
558 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
559 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200560 video_stream_encoder_->OnBitrateUpdated(
561 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +0100562 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100563
564 video_source_.IncomingCapturedFrame(
565 CreateFrame(1, codec_width_, codec_height_));
566 WaitForEncodedFrame(1);
567 }
568
asapersson02465b82017-04-10 01:12:52 -0700569 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700570 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700571 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
572 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700573 }
574
asapersson09f05612017-05-15 23:40:18 -0700575 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
576 const rtc::VideoSinkWants& wants2) {
577 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
578 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
579 }
580
Åsa Persson8c1bf952018-09-13 10:42:19 +0200581 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
582 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
583 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
584 EXPECT_FALSE(wants.target_pixel_count);
585 }
586
asapersson09f05612017-05-15 23:40:18 -0700587 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
588 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200589 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700590 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
591 EXPECT_GT(wants1.max_pixel_count, 0);
592 }
593
594 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
595 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200596 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700597 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
598 }
599
asaperssonf7e294d2017-06-13 23:25:22 -0700600 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
601 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200602 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700603 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
604 }
605
606 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
607 const rtc::VideoSinkWants& wants2) {
608 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
609 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
610 }
611
612 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
613 const rtc::VideoSinkWants& wants2) {
614 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
615 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
616 }
617
618 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
619 const rtc::VideoSinkWants& wants2) {
620 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
621 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
622 EXPECT_GT(wants1.max_pixel_count, 0);
623 }
624
625 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
626 const rtc::VideoSinkWants& wants2) {
627 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
628 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
629 }
630
asapersson09f05612017-05-15 23:40:18 -0700631 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
632 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200633 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700634 EXPECT_LT(wants.max_pixel_count, pixel_count);
635 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700636 }
637
638 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
639 EXPECT_LT(wants.max_framerate_fps, fps);
640 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
641 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700642 }
643
asaperssonf7e294d2017-06-13 23:25:22 -0700644 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
645 int expected_fps) {
646 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
647 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
648 EXPECT_FALSE(wants.target_pixel_count);
649 }
650
Jonathan Yubc771b72017-12-08 17:04:29 -0800651 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
652 int last_frame_pixels) {
653 // Balanced mode should always scale FPS to the desired range before
654 // attempting to scale resolution.
655 int fps_limit = wants.max_framerate_fps;
656 if (last_frame_pixels <= 320 * 240) {
657 EXPECT_TRUE(7 <= fps_limit && fps_limit <= 10);
658 } else if (last_frame_pixels <= 480 * 270) {
659 EXPECT_TRUE(10 <= fps_limit && fps_limit <= 15);
660 } else if (last_frame_pixels <= 640 * 480) {
661 EXPECT_LE(15, fps_limit);
662 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200663 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800664 }
665 }
666
sprang4847ae62017-06-27 07:06:52 -0700667 void WaitForEncodedFrame(int64_t expected_ntp_time) {
668 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100669 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700670 }
671
672 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
673 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100674 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700675 return ok;
676 }
677
678 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
679 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100680 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700681 }
682
683 void ExpectDroppedFrame() {
684 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100685 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700686 }
687
688 bool WaitForFrame(int64_t timeout_ms) {
689 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100690 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700691 return ok;
692 }
693
perkj26091b12016-09-01 01:17:40 -0700694 class TestEncoder : public test::FakeEncoder {
695 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100696 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700697
asaperssonfab67072017-04-04 05:51:49 -0700698 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800699 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700700 return config_;
701 }
702
703 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800704 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700705 block_next_encode_ = true;
706 }
707
Erik Språngaed30702018-11-05 12:57:17 +0100708 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800709 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100710 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100711 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100712 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100713 info.scaling_settings = VideoEncoder::ScalingSettings(
714 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100715 }
716 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100717 for (int i = 0; i < kMaxSpatialLayers; ++i) {
718 if (temporal_layers_supported_[i]) {
719 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
720 info.fps_allocation[i].resize(num_layers);
721 }
722 }
Erik Språngaed30702018-11-05 12:57:17 +0100723 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200724
725 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100726 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100727 return info;
kthelgason876222f2016-11-29 01:44:11 -0800728 }
729
Erik Språngb7cb7b52019-02-26 15:52:33 +0100730 int32_t RegisterEncodeCompleteCallback(
731 EncodedImageCallback* callback) override {
732 rtc::CritScope lock(&local_crit_sect_);
733 encoded_image_callback_ = callback;
734 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
735 }
736
perkjfa10b552016-10-02 23:45:26 -0700737 void ContinueEncode() { continue_encode_event_.Set(); }
738
739 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
740 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800741 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700742 EXPECT_EQ(timestamp_, timestamp);
743 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
744 }
745
kthelgason2fc52542017-03-03 00:24:41 -0800746 void SetQualityScaling(bool b) {
747 rtc::CritScope lock(&local_crit_sect_);
748 quality_scaling_ = b;
749 }
kthelgasonad9010c2017-02-14 00:46:51 -0800750
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100751 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
752 rtc::CritScope lock(&local_crit_sect_);
753 requested_resolution_alignment_ = requested_resolution_alignment;
754 }
755
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100756 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
757 rtc::CritScope lock(&local_crit_sect_);
758 is_hardware_accelerated_ = is_hardware_accelerated;
759 }
760
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100761 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
762 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
763 rtc::CritScope lock(&local_crit_sect_);
764 temporal_layers_supported_[spatial_idx] = supported;
765 }
766
Sergey Silkin6456e352019-07-08 17:56:40 +0200767 void SetResolutionBitrateLimits(
768 std::vector<ResolutionBitrateLimits> thresholds) {
769 rtc::CritScope cs(&local_crit_sect_);
770 resolution_bitrate_limits_ = thresholds;
771 }
772
sprangfe627f32017-03-29 08:24:59 -0700773 void ForceInitEncodeFailure(bool force_failure) {
774 rtc::CritScope lock(&local_crit_sect_);
775 force_init_encode_failed_ = force_failure;
776 }
777
Niels Möller6bb5ab92019-01-11 11:11:10 +0100778 void SimulateOvershoot(double rate_factor) {
779 rtc::CritScope lock(&local_crit_sect_);
780 rate_factor_ = rate_factor;
781 }
782
Erik Språngd7329ca2019-02-21 21:19:53 +0100783 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100784 rtc::CritScope lock(&local_crit_sect_);
785 return last_framerate_;
786 }
787
Erik Språngd7329ca2019-02-21 21:19:53 +0100788 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100789 rtc::CritScope lock(&local_crit_sect_);
790 return last_update_rect_;
791 }
792
Niels Möller87e2d782019-03-07 10:18:23 +0100793 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100794 rtc::CritScope lock(&local_crit_sect_);
795 return last_frame_types_;
796 }
797
798 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100799 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100800 keyframe ? VideoFrameType::kVideoFrameKey
801 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100802 {
803 rtc::CritScope lock(&local_crit_sect_);
804 last_frame_types_ = frame_type;
805 }
Niels Möllerb859b322019-03-07 12:40:01 +0100806 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100807 }
808
Erik Språngb7cb7b52019-02-26 15:52:33 +0100809 void InjectEncodedImage(const EncodedImage& image) {
810 rtc::CritScope lock(&local_crit_sect_);
811 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
812 }
813
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200814 void InjectEncodedImage(const EncodedImage& image,
815 const CodecSpecificInfo* codec_specific_info,
816 const RTPFragmentationHeader* fragmentation) {
817 rtc::CritScope lock(&local_crit_sect_);
818 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
819 fragmentation);
820 }
821
Erik Språngd7329ca2019-02-21 21:19:53 +0100822 void ExpectNullFrame() {
823 rtc::CritScope lock(&local_crit_sect_);
824 expect_null_frame_ = true;
825 }
826
Erik Språng5056af02019-09-02 15:53:11 +0200827 absl::optional<VideoEncoder::RateControlParameters>
828 GetAndResetLastRateControlSettings() {
829 auto settings = last_rate_control_settings_;
830 last_rate_control_settings_.reset();
831 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100832 }
833
Sergey Silkin5ee69672019-07-02 14:18:34 +0200834 int GetNumEncoderInitializations() const {
835 rtc::CritScope lock(&local_crit_sect_);
836 return num_encoder_initializations_;
837 }
838
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200839 int GetNumSetRates() const {
840 rtc::CritScope lock(&local_crit_sect_);
841 return num_set_rates_;
842 }
843
perkjfa10b552016-10-02 23:45:26 -0700844 private:
perkj26091b12016-09-01 01:17:40 -0700845 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100846 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700847 bool block_encode;
848 {
brandtre78d2662017-01-16 05:57:16 -0800849 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100850 if (expect_null_frame_) {
851 EXPECT_EQ(input_image.timestamp(), 0u);
852 EXPECT_EQ(input_image.width(), 1);
853 last_frame_types_ = *frame_types;
854 expect_null_frame_ = false;
855 } else {
856 EXPECT_GT(input_image.timestamp(), timestamp_);
857 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
858 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
859 }
perkj26091b12016-09-01 01:17:40 -0700860
861 timestamp_ = input_image.timestamp();
862 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700863 last_input_width_ = input_image.width();
864 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700865 block_encode = block_next_encode_;
866 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100867 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100868 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700869 }
Niels Möllerb859b322019-03-07 12:40:01 +0100870 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700871 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700872 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700873 return result;
874 }
875
sprangfe627f32017-03-29 08:24:59 -0700876 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200877 const Settings& settings) override {
878 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200879
sprangfe627f32017-03-29 08:24:59 -0700880 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100881 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200882
883 ++num_encoder_initializations_;
884
Erik Språng82fad3d2018-03-21 09:57:23 +0100885 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700886 // Simulate setting up temporal layers, in order to validate the life
887 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100888 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200889 frame_buffer_controller_ =
890 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700891 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100892 if (force_init_encode_failed_) {
893 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700894 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100895 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100896
Erik Språngb7cb7b52019-02-26 15:52:33 +0100897 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700898 return res;
899 }
900
Erik Språngb7cb7b52019-02-26 15:52:33 +0100901 int32_t Release() override {
902 rtc::CritScope lock(&local_crit_sect_);
903 EXPECT_NE(initialized_, EncoderState::kUninitialized);
904 initialized_ = EncoderState::kUninitialized;
905 return FakeEncoder::Release();
906 }
907
Erik Språng16cb8f52019-04-12 13:59:09 +0200908 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100909 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200910 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100911 VideoBitrateAllocation adjusted_rate_allocation;
912 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
913 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200914 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100915 adjusted_rate_allocation.SetBitrate(
916 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200917 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100918 rate_factor_));
919 }
920 }
921 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200922 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200923 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200924 RateControlParameters adjusted_paramters = parameters;
925 adjusted_paramters.bitrate = adjusted_rate_allocation;
926 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100927 }
928
brandtre78d2662017-01-16 05:57:16 -0800929 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100930 enum class EncoderState {
931 kUninitialized,
932 kInitializationFailed,
933 kInitialized
934 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
935 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700936 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700937 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700938 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
939 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
940 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
941 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
942 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100943 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100944 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100945 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700946 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100947 absl::optional<bool>
948 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
949 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700950 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100951 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
952 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200953 absl::optional<VideoEncoder::RateControlParameters>
954 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100955 VideoFrame::UpdateRect last_update_rect_
956 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100957 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100958 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100959 EncodedImageCallback* encoded_image_callback_
960 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 11:20:09 +0200961 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200962 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +0200963 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
964 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200965 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -0700966 };
967
mflodmancc3d4422017-08-03 08:27:51 -0700968 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -0700969 public:
970 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +0100971 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -0700972
perkj26091b12016-09-01 01:17:40 -0700973 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -0700974 EXPECT_TRUE(
975 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
976 }
977
978 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
979 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -0700980 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -0700981 if (!encoded_frame_event_.Wait(timeout_ms))
982 return false;
perkj26091b12016-09-01 01:17:40 -0700983 {
984 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -0800985 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -0700986 }
987 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -0700988 return true;
perkj26091b12016-09-01 01:17:40 -0700989 }
990
sprangb1ca0732017-02-01 08:38:12 -0800991 void WaitForEncodedFrame(uint32_t expected_width,
992 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -0700993 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100994 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -0700995 }
996
Åsa Perssonc74d8da2017-12-04 14:13:56 +0100997 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -0700998 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -0800999 uint32_t width = 0;
1000 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001001 {
1002 rtc::CritScope lock(&crit_);
1003 width = last_width_;
1004 height = last_height_;
1005 }
1006 EXPECT_EQ(expected_height, height);
1007 EXPECT_EQ(expected_width, width);
1008 }
1009
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001010 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1011 int width = 0;
1012 int height = 0;
1013 {
1014 rtc::CritScope lock(&crit_);
1015 width = last_width_;
1016 height = last_height_;
1017 }
1018 EXPECT_EQ(width % resolution_alignment, 0);
1019 EXPECT_EQ(height % resolution_alignment, 0);
1020 }
1021
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001022 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1023 VideoRotation rotation;
1024 {
1025 rtc::CritScope lock(&crit_);
1026 rotation = last_rotation_;
1027 }
1028 EXPECT_EQ(expected_rotation, rotation);
1029 }
1030
kthelgason2fc52542017-03-03 00:24:41 -08001031 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001032
sprangc5d62e22017-04-02 23:53:04 -07001033 bool WaitForFrame(int64_t timeout_ms) {
1034 return encoded_frame_event_.Wait(timeout_ms);
1035 }
1036
perkj26091b12016-09-01 01:17:40 -07001037 void SetExpectNoFrames() {
1038 rtc::CritScope lock(&crit_);
1039 expect_frames_ = false;
1040 }
1041
asaperssonfab67072017-04-04 05:51:49 -07001042 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001043 rtc::CritScope lock(&crit_);
1044 return number_of_reconfigurations_;
1045 }
1046
asaperssonfab67072017-04-04 05:51:49 -07001047 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001048 rtc::CritScope lock(&crit_);
1049 return min_transmit_bitrate_bps_;
1050 }
1051
Erik Språngd7329ca2019-02-21 21:19:53 +01001052 void SetNumExpectedLayers(size_t num_layers) {
1053 rtc::CritScope lock(&crit_);
1054 num_expected_layers_ = num_layers;
1055 }
1056
Erik Språngb7cb7b52019-02-26 15:52:33 +01001057 int64_t GetLastCaptureTimeMs() const {
1058 rtc::CritScope lock(&crit_);
1059 return last_capture_time_ms_;
1060 }
1061
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001062 std::vector<uint8_t> GetLastEncodedImageData() {
1063 rtc::CritScope lock(&crit_);
1064 return std::move(last_encoded_image_data_);
1065 }
1066
1067 RTPFragmentationHeader GetLastFragmentation() {
1068 rtc::CritScope lock(&crit_);
1069 return std::move(last_fragmentation_);
1070 }
1071
perkj26091b12016-09-01 01:17:40 -07001072 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001073 Result OnEncodedImage(
1074 const EncodedImage& encoded_image,
1075 const CodecSpecificInfo* codec_specific_info,
1076 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001077 rtc::CritScope lock(&crit_);
1078 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001079 last_encoded_image_data_ = std::vector<uint8_t>(
1080 encoded_image.data(), encoded_image.data() + encoded_image.size());
1081 if (fragmentation) {
1082 last_fragmentation_.CopyFrom(*fragmentation);
1083 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001084 uint32_t timestamp = encoded_image.Timestamp();
1085 if (last_timestamp_ != timestamp) {
1086 num_received_layers_ = 1;
1087 } else {
1088 ++num_received_layers_;
1089 }
1090 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001091 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001092 last_width_ = encoded_image._encodedWidth;
1093 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001094 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001095 if (num_received_layers_ == num_expected_layers_) {
1096 encoded_frame_event_.Set();
1097 }
sprangb1ca0732017-02-01 08:38:12 -08001098 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001099 }
1100
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001101 void OnEncoderConfigurationChanged(
1102 std::vector<VideoStream> streams,
1103 VideoEncoderConfig::ContentType content_type,
1104 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001105 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001106 ++number_of_reconfigurations_;
1107 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1108 }
1109
perkj26091b12016-09-01 01:17:40 -07001110 rtc::CriticalSection crit_;
1111 TestEncoder* test_encoder_;
1112 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001113 std::vector<uint8_t> last_encoded_image_data_;
1114 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001115 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001116 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001117 uint32_t last_height_ = 0;
1118 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001119 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001120 size_t num_expected_layers_ = 1;
1121 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001122 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001123 int number_of_reconfigurations_ = 0;
1124 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001125 };
1126
Sergey Silkin5ee69672019-07-02 14:18:34 +02001127 class VideoBitrateAllocatorProxyFactory
1128 : public VideoBitrateAllocatorFactory {
1129 public:
1130 VideoBitrateAllocatorProxyFactory()
1131 : bitrate_allocator_factory_(
1132 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1133
1134 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1135 const VideoCodec& codec) override {
1136 rtc::CritScope lock(&crit_);
1137 codec_config_ = codec;
1138 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1139 }
1140
1141 VideoCodec codec_config() const {
1142 rtc::CritScope lock(&crit_);
1143 return codec_config_;
1144 }
1145
1146 private:
1147 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1148
1149 rtc::CriticalSection crit_;
1150 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1151 };
1152
perkj26091b12016-09-01 01:17:40 -07001153 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001154 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001155 int codec_width_;
1156 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001157 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001158 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001159 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001160 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001161 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001162 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001163 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001164 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001165 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001166 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001167};
1168
mflodmancc3d4422017-08-03 08:27:51 -07001169TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001170 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001171 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001172 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001173 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001174 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001175 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001176 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001177 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001178}
1179
mflodmancc3d4422017-08-03 08:27:51 -07001180TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001181 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001182 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001183 // The encoder will cache up to one frame for a short duration. Adding two
1184 // frames means that the first frame will be dropped and the second frame will
1185 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001186 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001187 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001188 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001189
Erik Språng4c6ca302019-04-08 15:14:01 +02001190 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001191 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001192 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001193
Sebastian Janssona3177052018-04-10 13:05:49 +02001194 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001195 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001196 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1197
1198 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001199 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001200}
1201
mflodmancc3d4422017-08-03 08:27:51 -07001202TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001203 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001204 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001205 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001206 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001207 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001208
Florent Castellia8336d32019-09-09 13:36:55 +02001209 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
Ying Wang9b881ab2020-02-07 14:29:32 +01001210 DataRate::bps(0), 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001211 // The encoder will cache up to one frame for a short duration. Adding two
1212 // frames means that the first frame will be dropped and the second frame will
1213 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001214 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001215 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001216
Erik Språng4c6ca302019-04-08 15:14:01 +02001217 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001218 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001219 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001220 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001221 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1222 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001223 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001224}
1225
mflodmancc3d4422017-08-03 08:27:51 -07001226TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001227 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001228 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001229 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001230 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001231 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001232
1233 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001234 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001235
perkja49cbd32016-09-16 07:53:41 -07001236 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001237 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001238 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001239}
1240
mflodmancc3d4422017-08-03 08:27:51 -07001241TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001242 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001243 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001244 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001245
perkja49cbd32016-09-16 07:53:41 -07001246 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001247 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001248
mflodmancc3d4422017-08-03 08:27:51 -07001249 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001250 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001251 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001252 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1253 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001254}
1255
mflodmancc3d4422017-08-03 08:27:51 -07001256TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001257 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001258 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001259 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001260
1261 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001262 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001263 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001264 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1265 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001266 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1267 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001268 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001269 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001270
mflodmancc3d4422017-08-03 08:27:51 -07001271 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001272}
1273
Noah Richards51db4212019-06-12 06:59:12 -07001274TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1275 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001276 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001277 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001278
1279 rtc::Event frame_destroyed_event;
1280 video_source_.IncomingCapturedFrame(
1281 CreateFakeNativeFrame(1, &frame_destroyed_event));
1282 ExpectDroppedFrame();
1283 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1284 video_stream_encoder_->Stop();
1285}
1286
1287TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1288 // Use the cropping factory.
1289 video_encoder_config_.video_stream_factory =
1290 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1291 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1292 kMaxPayloadLength);
1293 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1294
1295 // Capture a frame at codec_width_/codec_height_.
1296 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001297 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001298 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001299 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1300 WaitForEncodedFrame(1);
1301 // The encoder will have been configured once.
1302 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1303 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1304 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1305
1306 // Now send in a fake frame that needs to be cropped as the width/height
1307 // aren't divisible by 4 (see CreateEncoderStreams above).
1308 rtc::Event frame_destroyed_event;
1309 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1310 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1311 ExpectDroppedFrame();
1312 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1313 video_stream_encoder_->Stop();
1314}
1315
Ying Wang9b881ab2020-02-07 14:29:32 +01001316TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
1317 video_stream_encoder_->OnBitrateUpdated(
1318 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1319 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
1320 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1321 WaitForEncodedFrame(1);
1322
1323 video_stream_encoder_->OnBitrateUpdated(
1324 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
1325 DataRate::bps(kTargetBitrateBps), 0, 0, 0.5);
1326 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1327 // frames. Adding two frames means that the first frame will be dropped and
1328 // the second frame will be sent to the encoder.
1329 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1330 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1331 WaitForEncodedFrame(3);
1332 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1333 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1334 WaitForEncodedFrame(5);
1335 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1336 video_stream_encoder_->Stop();
1337}
1338
mflodmancc3d4422017-08-03 08:27:51 -07001339TEST_F(VideoStreamEncoderTest,
1340 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001341 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001342 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001343 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001344 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001345
1346 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001347 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001348 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001349 // The encoder will have been configured once when the first frame is
1350 // received.
1351 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001352
1353 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001354 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001355 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001356 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001357 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001358
1359 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001360 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001361 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001362 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001363 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001364
mflodmancc3d4422017-08-03 08:27:51 -07001365 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001366}
1367
mflodmancc3d4422017-08-03 08:27:51 -07001368TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001369 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001370 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001371 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001372
1373 // Capture a frame and wait for it to synchronize with the encoder thread.
1374 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001375 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001376 // The encoder will have been configured once.
1377 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001378 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1379 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1380
1381 codec_width_ *= 2;
1382 codec_height_ *= 2;
1383 // Capture a frame with a higher resolution and wait for it to synchronize
1384 // with the encoder thread.
1385 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001386 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001387 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1388 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001389 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001390
mflodmancc3d4422017-08-03 08:27:51 -07001391 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001392}
1393
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001394TEST_F(VideoStreamEncoderTest,
1395 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1396 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001397 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001398 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001399
1400 // Capture a frame and wait for it to synchronize with the encoder thread.
1401 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1402 WaitForEncodedFrame(1);
1403
1404 VideoEncoderConfig video_encoder_config;
1405 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1406 // Changing the max payload data length recreates encoder.
1407 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1408 kMaxPayloadLength / 2);
1409
1410 // Capture a frame and wait for it to synchronize with the encoder thread.
1411 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1412 WaitForEncodedFrame(2);
1413 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1414
1415 video_stream_encoder_->Stop();
1416}
1417
Sergey Silkin5ee69672019-07-02 14:18:34 +02001418TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1419 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001420 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001421 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001422
1423 VideoEncoderConfig video_encoder_config;
1424 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1425 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1426 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1427 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1428 kMaxPayloadLength);
1429
1430 // Capture a frame and wait for it to synchronize with the encoder thread.
1431 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1432 WaitForEncodedFrame(1);
1433 // The encoder will have been configured once when the first frame is
1434 // received.
1435 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1436 EXPECT_EQ(kTargetBitrateBps,
1437 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1438 EXPECT_EQ(kStartBitrateBps,
1439 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1440
Sergey Silkin6456e352019-07-08 17:56:40 +02001441 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1442 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001443 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1444 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1445 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1446 kMaxPayloadLength);
1447
1448 // Capture a frame and wait for it to synchronize with the encoder thread.
1449 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1450 WaitForEncodedFrame(2);
1451 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1452 // Bitrate limits have changed - rate allocator should be reconfigured,
1453 // encoder should not be reconfigured.
1454 EXPECT_EQ(kTargetBitrateBps * 2,
1455 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1456 EXPECT_EQ(kStartBitrateBps * 2,
1457 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1458 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1459
1460 video_stream_encoder_->Stop();
1461}
1462
Sergey Silkin6456e352019-07-08 17:56:40 +02001463TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001464 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001465 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001466 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001467 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001468
Sergey Silkincd02eba2020-01-20 14:48:40 +01001469 const uint32_t kMinEncBitrateKbps = 100;
1470 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001471 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001472 /*frame_size_pixels=*/codec_width_ * codec_height_,
1473 /*min_start_bitrate_bps=*/0,
1474 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1475 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001476 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1477
Sergey Silkincd02eba2020-01-20 14:48:40 +01001478 VideoEncoderConfig video_encoder_config;
1479 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1480 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1481 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1482 (kMinEncBitrateKbps + 1) * 1000;
1483 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1484 kMaxPayloadLength);
1485
1486 // When both encoder and app provide bitrate limits, the intersection of
1487 // provided sets should be used.
1488 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1489 WaitForEncodedFrame(1);
1490 EXPECT_EQ(kMaxEncBitrateKbps,
1491 bitrate_allocator_factory_.codec_config().maxBitrate);
1492 EXPECT_EQ(kMinEncBitrateKbps + 1,
1493 bitrate_allocator_factory_.codec_config().minBitrate);
1494
1495 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1496 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1497 (kMinEncBitrateKbps - 1) * 1000;
1498 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1499 kMaxPayloadLength);
1500 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001501 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001502 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001503 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001504 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001505 bitrate_allocator_factory_.codec_config().minBitrate);
1506
Sergey Silkincd02eba2020-01-20 14:48:40 +01001507 video_stream_encoder_->Stop();
1508}
1509
1510TEST_F(VideoStreamEncoderTest,
1511 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
1512 video_stream_encoder_->OnBitrateUpdated(
1513 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001514 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001515
1516 const uint32_t kMinAppBitrateKbps = 100;
1517 const uint32_t kMaxAppBitrateKbps = 200;
1518 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1519 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1520 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1521 /*frame_size_pixels=*/codec_width_ * codec_height_,
1522 /*min_start_bitrate_bps=*/0,
1523 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1524 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1525 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1526
1527 VideoEncoderConfig video_encoder_config;
1528 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1529 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1530 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1531 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001532 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1533 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001534
Sergey Silkincd02eba2020-01-20 14:48:40 +01001535 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1536 WaitForEncodedFrame(1);
1537 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001538 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001539 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001540 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001541
1542 video_stream_encoder_->Stop();
1543}
1544
1545TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001546 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001547 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001548 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001549 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001550
1551 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001552 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001553 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001554 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001555 fake_encoder_.SetResolutionBitrateLimits(
1556 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1557
1558 VideoEncoderConfig video_encoder_config;
1559 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1560 video_encoder_config.max_bitrate_bps = 0;
1561 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1562 kMaxPayloadLength);
1563
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001564 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001565 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1566 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001567 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1568 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001569 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1570 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1571
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001572 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001573 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1574 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001575 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1576 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001577 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1578 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1579
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001580 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001581 // encoder for 360p should be used.
1582 video_source_.IncomingCapturedFrame(
1583 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1584 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001585 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1586 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001587 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1588 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1589
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001590 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001591 // ignored.
1592 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1593 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001594 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1595 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001596 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1597 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001598 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1599 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001600 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1601 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1602
1603 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1604 // for 270p should be used.
1605 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1606 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001607 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1608 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001609 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1610 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1611
1612 video_stream_encoder_->Stop();
1613}
1614
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001615TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1616 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001617 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001618 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001619
1620 VideoEncoderConfig video_encoder_config;
1621 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1622 video_encoder_config.max_bitrate_bps = 0;
1623 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1624 kMaxPayloadLength);
1625
1626 // Encode 720p frame to get the default encoder target bitrate.
1627 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1628 WaitForEncodedFrame(1);
1629 const uint32_t kDefaultTargetBitrateFor720pKbps =
1630 bitrate_allocator_factory_.codec_config()
1631 .simulcastStream[0]
1632 .targetBitrate;
1633
1634 // Set the max recommended encoder bitrate to something lower than the default
1635 // target bitrate.
1636 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1637 1280 * 720, 10 * 1000, 10 * 1000,
1638 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1639 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1640
1641 // Change resolution to trigger encoder reinitialization.
1642 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1643 WaitForEncodedFrame(2);
1644 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1645 WaitForEncodedFrame(3);
1646
1647 // Ensure the target bitrate is capped by the max bitrate.
1648 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1649 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1650 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1651 .simulcastStream[0]
1652 .targetBitrate *
1653 1000,
1654 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1655
1656 video_stream_encoder_->Stop();
1657}
1658
mflodmancc3d4422017-08-03 08:27:51 -07001659TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001660 EXPECT_TRUE(video_source_.has_sinks());
1661 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001662 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001663 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001664 EXPECT_FALSE(video_source_.has_sinks());
1665 EXPECT_TRUE(new_video_source.has_sinks());
1666
mflodmancc3d4422017-08-03 08:27:51 -07001667 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001668}
1669
mflodmancc3d4422017-08-03 08:27:51 -07001670TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001671 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001672 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001673 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001674 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001675}
1676
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001677TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1678 constexpr int kRequestedResolutionAlignment = 7;
1679 video_source_.set_adaptation_enabled(true);
1680 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
1681 video_stream_encoder_->OnBitrateUpdated(
1682 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001683 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001684
1685 // On the 1st frame, we should have initialized the encoder and
1686 // asked for its resolution requirements.
1687 video_source_.IncomingCapturedFrame(
1688 CreateFrame(1, codec_width_, codec_height_));
1689 WaitForEncodedFrame(1);
1690 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1691 kRequestedResolutionAlignment);
1692
1693 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1694 // (It's up the to the encoder to potentially drop the previous frame,
1695 // to avoid coding back-to-back keyframes.)
1696 video_source_.IncomingCapturedFrame(
1697 CreateFrame(2, codec_width_, codec_height_));
1698 WaitForEncodedFrame(2);
1699 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1700
1701 video_stream_encoder_->Stop();
1702}
1703
Jonathan Yubc771b72017-12-08 17:04:29 -08001704TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1705 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001706 const int kWidth = 1280;
1707 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001708
1709 // We rely on the automatic resolution adaptation, but we handle framerate
1710 // adaptation manually by mocking the stats proxy.
1711 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001712
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001713 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001714 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001715 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001716 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001717 video_stream_encoder_->SetSource(&video_source_,
1718 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001719 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001721 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001722 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1723
Jonathan Yubc771b72017-12-08 17:04:29 -08001724 // Adapt down as far as possible.
1725 rtc::VideoSinkWants last_wants;
1726 int64_t t = 1;
1727 int loop_count = 0;
1728 do {
1729 ++loop_count;
1730 last_wants = video_source_.sink_wants();
1731
1732 // Simulate the framerate we've been asked to adapt to.
1733 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1734 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1735 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1736 mock_stats.input_frame_rate = fps;
1737 stats_proxy_->SetMockStats(mock_stats);
1738
1739 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1740 sink_.WaitForEncodedFrame(t);
1741 t += frame_interval_ms;
1742
mflodmancc3d4422017-08-03 08:27:51 -07001743 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001744 VerifyBalancedModeFpsRange(
1745 video_source_.sink_wants(),
1746 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1747 } while (video_source_.sink_wants().max_pixel_count <
1748 last_wants.max_pixel_count ||
1749 video_source_.sink_wants().max_framerate_fps <
1750 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001751
Jonathan Yubc771b72017-12-08 17:04:29 -08001752 // Verify that we've adapted all the way down.
1753 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001754 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001755 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1756 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001757 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001758 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1759 *video_source_.last_sent_height());
1760 EXPECT_EQ(kMinBalancedFramerateFps,
1761 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001762
Jonathan Yubc771b72017-12-08 17:04:29 -08001763 // Adapt back up the same number of times we adapted down.
1764 for (int i = 0; i < loop_count - 1; ++i) {
1765 last_wants = video_source_.sink_wants();
1766
1767 // Simulate the framerate we've been asked to adapt to.
1768 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1769 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1770 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1771 mock_stats.input_frame_rate = fps;
1772 stats_proxy_->SetMockStats(mock_stats);
1773
1774 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1775 sink_.WaitForEncodedFrame(t);
1776 t += frame_interval_ms;
1777
mflodmancc3d4422017-08-03 08:27:51 -07001778 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001779 VerifyBalancedModeFpsRange(
1780 video_source_.sink_wants(),
1781 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1782 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1783 last_wants.max_pixel_count ||
1784 video_source_.sink_wants().max_framerate_fps >
1785 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001786 }
1787
Åsa Persson8c1bf952018-09-13 10:42:19 +02001788 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001789 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001790 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1792 EXPECT_EQ((loop_count - 1) * 2,
1793 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001794
mflodmancc3d4422017-08-03 08:27:51 -07001795 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001796}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001797
mflodmancc3d4422017-08-03 08:27:51 -07001798TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001799 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001800 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001801 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001802 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001803
sprangc5d62e22017-04-02 23:53:04 -07001804 const int kFrameWidth = 1280;
1805 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001806
Åsa Persson8c1bf952018-09-13 10:42:19 +02001807 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001808
kthelgason5e13d412016-12-01 03:59:51 -08001809 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001810 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001811 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001812 frame_timestamp += kFrameIntervalMs;
1813
perkj803d97f2016-11-01 11:45:46 -07001814 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001815 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001816 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001817 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001818 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001819 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001820
asapersson0944a802017-04-07 00:57:58 -07001821 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001822 // wanted resolution.
1823 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1824 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1825 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001826 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001827
1828 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001829 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001830 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001831 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001832 // Give the encoder queue time to process the change in degradation preference
1833 // by waiting for an encoded frame.
1834 new_video_source.IncomingCapturedFrame(
1835 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1836 sink_.WaitForEncodedFrame(frame_timestamp);
1837 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001838 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001839 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001840
sprangc5d62e22017-04-02 23:53:04 -07001841 // Force an input frame rate to be available, or the adaptation call won't
1842 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001843 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001844 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001845 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001846 stats_proxy_->SetMockStats(stats);
1847
mflodmancc3d4422017-08-03 08:27:51 -07001848 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001849 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001850 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001851 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001852 frame_timestamp += kFrameIntervalMs;
1853
1854 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001855 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001856 EXPECT_EQ(std::numeric_limits<int>::max(),
1857 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001858 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001859
asapersson02465b82017-04-10 01:12:52 -07001860 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001861 video_stream_encoder_->SetSource(&new_video_source,
1862 webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01001863 // Give the encoder queue time to process the change in degradation preference
1864 // by waiting for an encoded frame.
1865 new_video_source.IncomingCapturedFrame(
1866 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1867 sink_.WaitForEncodedFrame(frame_timestamp);
1868 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001869 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001870
mflodmancc3d4422017-08-03 08:27:51 -07001871 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001872 new_video_source.IncomingCapturedFrame(
1873 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001874 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001875 frame_timestamp += kFrameIntervalMs;
1876
1877 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001878 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001879
1880 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001881 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001882 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01001883 // Give the encoder queue time to process the change in degradation preference
1884 // by waiting for an encoded frame.
1885 new_video_source.IncomingCapturedFrame(
1886 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1887 sink_.WaitForEncodedFrame(frame_timestamp);
1888 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001889 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1890 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001891 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001892 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001893
1894 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001895 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001896 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001897 // Give the encoder queue time to process the change in degradation preference
1898 // by waiting for an encoded frame.
1899 new_video_source.IncomingCapturedFrame(
1900 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1901 sink_.WaitForEncodedFrame(frame_timestamp);
1902 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001903 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1904 EXPECT_EQ(std::numeric_limits<int>::max(),
1905 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001906 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001907
mflodmancc3d4422017-08-03 08:27:51 -07001908 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001909}
1910
mflodmancc3d4422017-08-03 08:27:51 -07001911TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001912 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001913 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001914 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001915
asaperssonfab67072017-04-04 05:51:49 -07001916 const int kWidth = 1280;
1917 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001918 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001919 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001920 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1921 EXPECT_FALSE(stats.bw_limited_resolution);
1922 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1923
1924 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001925 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001926 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001927 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001928
1929 stats = stats_proxy_->GetStats();
1930 EXPECT_TRUE(stats.bw_limited_resolution);
1931 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1932
1933 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001934 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001935 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001936 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001937
1938 stats = stats_proxy_->GetStats();
1939 EXPECT_FALSE(stats.bw_limited_resolution);
1940 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1941 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1942
mflodmancc3d4422017-08-03 08:27:51 -07001943 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07001944}
1945
mflodmancc3d4422017-08-03 08:27:51 -07001946TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001947 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001948 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001949 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07001950
1951 const int kWidth = 1280;
1952 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001953 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001954 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07001955 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1956 EXPECT_FALSE(stats.cpu_limited_resolution);
1957 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1958
1959 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001960 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001961 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001962 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07001963
1964 stats = stats_proxy_->GetStats();
1965 EXPECT_TRUE(stats.cpu_limited_resolution);
1966 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
1967
1968 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07001969 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07001970 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001971 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07001972
1973 stats = stats_proxy_->GetStats();
1974 EXPECT_FALSE(stats.cpu_limited_resolution);
1975 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07001976 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07001977
mflodmancc3d4422017-08-03 08:27:51 -07001978 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001979}
1980
mflodmancc3d4422017-08-03 08:27:51 -07001981TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001982 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02001983 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01001984 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08001985
asaperssonfab67072017-04-04 05:51:49 -07001986 const int kWidth = 1280;
1987 const int kHeight = 720;
1988 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001989 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08001990 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07001991 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08001992 EXPECT_FALSE(stats.cpu_limited_resolution);
1993 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1994
asaperssonfab67072017-04-04 05:51:49 -07001995 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001996 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07001997 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001998 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08001999 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002000 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002001 EXPECT_TRUE(stats.cpu_limited_resolution);
2002 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2003
2004 // Set new source with adaptation still enabled.
2005 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002007 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002008
asaperssonfab67072017-04-04 05:51:49 -07002009 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002010 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002011 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002012 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002013 EXPECT_TRUE(stats.cpu_limited_resolution);
2014 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2015
2016 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002017 video_stream_encoder_->SetSource(&new_video_source,
2018 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002019
asaperssonfab67072017-04-04 05:51:49 -07002020 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002021 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002022 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002023 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002024 EXPECT_FALSE(stats.cpu_limited_resolution);
2025 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2026
2027 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002028 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002029 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002030
asaperssonfab67072017-04-04 05:51:49 -07002031 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002032 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002033 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002034 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002035 EXPECT_TRUE(stats.cpu_limited_resolution);
2036 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2037
asaperssonfab67072017-04-04 05:51:49 -07002038 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07002039 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002040 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002041 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002042 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002043 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002044 EXPECT_FALSE(stats.cpu_limited_resolution);
2045 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002046 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002047
mflodmancc3d4422017-08-03 08:27:51 -07002048 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002049}
2050
mflodmancc3d4422017-08-03 08:27:51 -07002051TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002052 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002053 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002054 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002055
asaperssonfab67072017-04-04 05:51:49 -07002056 const int kWidth = 1280;
2057 const int kHeight = 720;
2058 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002059 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002060 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002061 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002062 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002063 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002064
2065 // Set new source with adaptation still enabled.
2066 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002067 video_stream_encoder_->SetSource(&new_video_source,
2068 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002069
asaperssonfab67072017-04-04 05:51:49 -07002070 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002071 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002072 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002073 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002074 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002075 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002076
asaperssonfab67072017-04-04 05:51:49 -07002077 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002078 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002079 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002080 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002081 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002082 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002083 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002084 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002085
asaperssonfab67072017-04-04 05:51:49 -07002086 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002087 video_stream_encoder_->SetSource(&new_video_source,
2088 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002089
asaperssonfab67072017-04-04 05:51:49 -07002090 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002091 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002092 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002093 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002094 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002095 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002096
asapersson02465b82017-04-10 01:12:52 -07002097 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002098 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002099 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002100
asaperssonfab67072017-04-04 05:51:49 -07002101 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002102 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002103 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002104 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002105 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002106 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2107 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002108
mflodmancc3d4422017-08-03 08:27:51 -07002109 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002110}
2111
mflodmancc3d4422017-08-03 08:27:51 -07002112TEST_F(VideoStreamEncoderTest,
2113 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002114 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002115 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002116 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002117
2118 const int kWidth = 1280;
2119 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002120 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002121 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002122 video_source_.IncomingCapturedFrame(
2123 CreateFrame(timestamp_ms, kWidth, kHeight));
2124 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002125 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2126 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2127 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2128
2129 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002130 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002131 timestamp_ms += kFrameIntervalMs;
2132 video_source_.IncomingCapturedFrame(
2133 CreateFrame(timestamp_ms, kWidth, kHeight));
2134 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002135 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2136 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2137 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2138
2139 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002140 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002141 timestamp_ms += kFrameIntervalMs;
2142 video_source_.IncomingCapturedFrame(
2143 CreateFrame(timestamp_ms, kWidth, kHeight));
2144 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002145 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2146 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2147 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2148
Niels Möller4db138e2018-04-19 09:04:13 +02002149 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002150 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002151
2152 VideoEncoderConfig video_encoder_config;
2153 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2154 // Make format different, to force recreation of encoder.
2155 video_encoder_config.video_format.parameters["foo"] = "foo";
2156 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002157 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002158 timestamp_ms += kFrameIntervalMs;
2159 video_source_.IncomingCapturedFrame(
2160 CreateFrame(timestamp_ms, kWidth, kHeight));
2161 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002162 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2163 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2164 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2165
mflodmancc3d4422017-08-03 08:27:51 -07002166 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002167}
2168
mflodmancc3d4422017-08-03 08:27:51 -07002169TEST_F(VideoStreamEncoderTest,
2170 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002171 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002172 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002173 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002174
asapersson0944a802017-04-07 00:57:58 -07002175 const int kWidth = 1280;
2176 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002177 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002178
asaperssonfab67072017-04-04 05:51:49 -07002179 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002180 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002181 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002182 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002183 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002184 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2185
asapersson02465b82017-04-10 01:12:52 -07002186 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002187 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002188 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002189 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002190 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002191 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002192 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002193 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2194
2195 // Set new source with adaptation still enabled.
2196 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002197 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002198 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002199
2200 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002201 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002202 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002203 stats = stats_proxy_->GetStats();
2204 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002205 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002206 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2207
sprangc5d62e22017-04-02 23:53:04 -07002208 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002209 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002210 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002211 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002212 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002213 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002214 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002215 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002216 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002217 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002218 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2219
sprangc5d62e22017-04-02 23:53:04 -07002220 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002221 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002222 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2223 mock_stats.input_frame_rate = 30;
2224 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002225 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002226 stats_proxy_->ResetMockStats();
2227
2228 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002229 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002230 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002231
2232 // Framerate now adapted.
2233 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002234 EXPECT_FALSE(stats.cpu_limited_resolution);
2235 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002236 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2237
2238 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002239 video_stream_encoder_->SetSource(&new_video_source,
2240 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002241 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002242 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002243 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002244
2245 stats = stats_proxy_->GetStats();
2246 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002247 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002248 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2249
2250 // Try to trigger overuse. Should not succeed.
2251 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002252 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002253 stats_proxy_->ResetMockStats();
2254
2255 stats = stats_proxy_->GetStats();
2256 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002257 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002258 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2259
2260 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002261 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002262 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002263 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002264 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002265 stats = stats_proxy_->GetStats();
2266 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002267 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002268 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002269
2270 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002271 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002272 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002273 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002274 stats = stats_proxy_->GetStats();
2275 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002276 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002277 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2278
2279 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002280 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002281 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002282 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002283 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002284 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002285 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002286 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002287 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002288 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002289 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2290
2291 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002292 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002293 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002294 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002295 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002296 stats = stats_proxy_->GetStats();
2297 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002298 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002299 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002300 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002301
mflodmancc3d4422017-08-03 08:27:51 -07002302 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002303}
2304
mflodmancc3d4422017-08-03 08:27:51 -07002305TEST_F(VideoStreamEncoderTest,
2306 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002307 const int kWidth = 1280;
2308 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002309 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002310 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002311 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002312
asaperssonfab67072017-04-04 05:51:49 -07002313 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002314 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002315
asaperssonfab67072017-04-04 05:51:49 -07002316 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002317 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002318
asaperssonfab67072017-04-04 05:51:49 -07002319 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002320 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002321
asaperssonfab67072017-04-04 05:51:49 -07002322 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002323 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002324
kthelgason876222f2016-11-29 01:44:11 -08002325 // Expect a scale down.
2326 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002327 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002328
asapersson02465b82017-04-10 01:12:52 -07002329 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002330 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002331 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002332 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002333
asaperssonfab67072017-04-04 05:51:49 -07002334 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002335 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002336 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002337 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002338
asaperssonfab67072017-04-04 05:51:49 -07002339 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002340 EXPECT_EQ(std::numeric_limits<int>::max(),
2341 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002342
asaperssonfab67072017-04-04 05:51:49 -07002343 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002344 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002345 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002346 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002347
asapersson02465b82017-04-10 01:12:52 -07002348 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002349 EXPECT_EQ(std::numeric_limits<int>::max(),
2350 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002351
mflodmancc3d4422017-08-03 08:27:51 -07002352 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002353}
2354
mflodmancc3d4422017-08-03 08:27:51 -07002355TEST_F(VideoStreamEncoderTest,
2356 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002357 const int kWidth = 1280;
2358 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002359 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002360 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002361 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002362
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002363 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002364 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002365 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002366 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002367
2368 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002369 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002370 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002371 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2372 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2373
2374 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002375 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002376 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002377 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2378 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2379 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2380
2381 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002382 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002383 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2384 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2385 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2386
mflodmancc3d4422017-08-03 08:27:51 -07002387 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002388}
2389
mflodmancc3d4422017-08-03 08:27:51 -07002390TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002391 const int kWidth = 1280;
2392 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002393 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002394 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002395 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002396
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002397 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002398 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002399 video_stream_encoder_->SetSource(&source,
2400 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002401 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2402 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002403 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002404
2405 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002406 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002407 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2408 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2409 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2410 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2411
2412 // Trigger adapt down for same input resolution, expect no change.
2413 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2414 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002415 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002416 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2417 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2418 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2419
2420 // Trigger adapt down for larger input resolution, expect no change.
2421 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2422 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002423 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002424 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2426 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2427
mflodmancc3d4422017-08-03 08:27:51 -07002428 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002429}
2430
mflodmancc3d4422017-08-03 08:27:51 -07002431TEST_F(VideoStreamEncoderTest,
2432 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002433 const int kWidth = 1280;
2434 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002435 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002436 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002437 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002438
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002439 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002440 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002441 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002442 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002443
2444 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002445 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002446 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002447 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2448 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2449
2450 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002451 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002452 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002453 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2454 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2455
mflodmancc3d4422017-08-03 08:27:51 -07002456 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002457}
2458
mflodmancc3d4422017-08-03 08:27:51 -07002459TEST_F(VideoStreamEncoderTest,
2460 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002461 const int kWidth = 1280;
2462 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002463 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002464 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002465 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002466
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002467 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002468 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002469 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002470 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002471
2472 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002473 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002474 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002475 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002476 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2477
2478 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002479 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002480 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002481 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002482 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2483
mflodmancc3d4422017-08-03 08:27:51 -07002484 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002485}
2486
mflodmancc3d4422017-08-03 08:27:51 -07002487TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002488 const int kWidth = 1280;
2489 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002490 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002491 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002492 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002493
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002494 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002495 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002496 video_stream_encoder_->SetSource(&source,
2497 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002498
2499 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2500 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002501 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2503 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2504 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2505
2506 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002507 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002508 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002509 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2510 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2511 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2512
mflodmancc3d4422017-08-03 08:27:51 -07002513 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002514}
2515
mflodmancc3d4422017-08-03 08:27:51 -07002516TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002517 const int kWidth = 1280;
2518 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002519 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002520 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002521 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002522
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002523 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002524 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002525 video_stream_encoder_->SetSource(&source,
2526 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002527
2528 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2529 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002530 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002531 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2533 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2534
2535 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002536 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002537 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2539 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2540 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2541
mflodmancc3d4422017-08-03 08:27:51 -07002542 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002543}
2544
mflodmancc3d4422017-08-03 08:27:51 -07002545TEST_F(VideoStreamEncoderTest,
2546 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002547 const int kWidth = 1280;
2548 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002549 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002550 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002551 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002552
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002554 AdaptingFrameForwarder source;
2555 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002556 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002557 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002558
2559 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002560 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002561 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002562 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2563 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2564
2565 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002566 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002567 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002568 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002569 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2571 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2572
2573 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002574 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002575 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002576 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2577 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2578 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2579
mflodmancc3d4422017-08-03 08:27:51 -07002580 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002581}
2582
mflodmancc3d4422017-08-03 08:27:51 -07002583TEST_F(VideoStreamEncoderTest,
2584 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002585 const int kWidth = 1280;
2586 const int kHeight = 720;
2587 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002588 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002589 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002590 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002591
2592 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2593 stats.input_frame_rate = kInputFps;
2594 stats_proxy_->SetMockStats(stats);
2595
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002596 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002597 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2598 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002599 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002600
2601 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002602 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002603 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2604 sink_.WaitForEncodedFrame(2);
2605 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2606
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002607 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002608 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002609 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002610 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002611 // Give the encoder queue time to process the change in degradation preference
2612 // by waiting for an encoded frame.
2613 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2614 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002615 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002616
2617 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002618 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002619 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2620 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002621 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2622
2623 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002624 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002625 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002626
mflodmancc3d4422017-08-03 08:27:51 -07002627 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002628}
2629
mflodmancc3d4422017-08-03 08:27:51 -07002630TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002631 const int kWidth = 1280;
2632 const int kHeight = 720;
2633 const size_t kNumFrames = 10;
2634
Erik Språng4c6ca302019-04-08 15:14:01 +02002635 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002636 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002637 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002638
asaperssond0de2952017-04-21 01:47:31 -07002639 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002640 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002641 video_source_.set_adaptation_enabled(true);
2642
2643 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2644 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2645
2646 int downscales = 0;
2647 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002648 video_source_.IncomingCapturedFrame(
2649 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2650 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002651
asaperssonfab67072017-04-04 05:51:49 -07002652 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002653 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002654 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002655 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002656
2657 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2658 ++downscales;
2659
2660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2661 EXPECT_EQ(downscales,
2662 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2663 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002664 }
mflodmancc3d4422017-08-03 08:27:51 -07002665 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002666}
2667
mflodmancc3d4422017-08-03 08:27:51 -07002668TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002669 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2670 const int kWidth = 1280;
2671 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002672 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002673 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002674 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002675
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002676 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002677 AdaptingFrameForwarder source;
2678 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002679 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002680 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002681
Åsa Persson8c1bf952018-09-13 10:42:19 +02002682 int64_t timestamp_ms = kFrameIntervalMs;
2683 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002684 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002685 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002686 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2687 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2688
2689 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002690 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002691 timestamp_ms += kFrameIntervalMs;
2692 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2693 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002694 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002695 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2696 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2697
2698 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002699 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002700 timestamp_ms += kFrameIntervalMs;
2701 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002702 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002703 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002704 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2705 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2706
2707 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002709 timestamp_ms += kFrameIntervalMs;
2710 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2711 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002712 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002713 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2714 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2715
2716 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002717 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002718 timestamp_ms += kFrameIntervalMs;
2719 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002720 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002721 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002722 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2723 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2724
mflodmancc3d4422017-08-03 08:27:51 -07002725 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002726}
2727
mflodmancc3d4422017-08-03 08:27:51 -07002728TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002729 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2730 const int kWidth = 1280;
2731 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002732 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02002733 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002734 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002735
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002736 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002737 AdaptingFrameForwarder source;
2738 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002739 video_stream_encoder_->SetSource(&source,
2740 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002741
Åsa Persson8c1bf952018-09-13 10:42:19 +02002742 int64_t timestamp_ms = kFrameIntervalMs;
2743 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002744 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002745 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2747 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2748
2749 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002750 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002751 timestamp_ms += kFrameIntervalMs;
2752 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2753 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002754 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2755 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2756 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2757
2758 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002759 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002760 timestamp_ms += kFrameIntervalMs;
2761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002762 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002763 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002764 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2765 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2766
2767 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002768 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002769 timestamp_ms += kFrameIntervalMs;
2770 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2771 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002772 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2773 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2774 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2775
2776 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002777 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002778 timestamp_ms += kFrameIntervalMs;
2779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002780 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002781 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2783 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2784
mflodmancc3d4422017-08-03 08:27:51 -07002785 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002786}
2787
Sergey Silkin41c650b2019-10-14 13:12:19 +02002788TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2789 fake_encoder_.SetResolutionBitrateLimits(
2790 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2791
2792 video_stream_encoder_->OnBitrateUpdated(
2793 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2794 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002795 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002796
2797 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2798 AdaptingFrameForwarder source;
2799 source.set_adaptation_enabled(true);
2800 video_stream_encoder_->SetSource(
2801 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2802
2803 // Insert 720p frame.
2804 int64_t timestamp_ms = kFrameIntervalMs;
2805 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2806 WaitForEncodedFrame(1280, 720);
2807
2808 // Reduce bitrate and trigger adapt down.
2809 video_stream_encoder_->OnBitrateUpdated(
2810 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2811 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002812 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002813 video_stream_encoder_->TriggerQualityLow();
2814
2815 // Insert 720p frame. It should be downscaled and encoded.
2816 timestamp_ms += kFrameIntervalMs;
2817 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2818 WaitForEncodedFrame(960, 540);
2819
2820 // Trigger adapt up. Higher resolution should not be requested duo to lack
2821 // of bitrate.
2822 video_stream_encoder_->TriggerQualityHigh();
2823 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2824
2825 // Increase bitrate.
2826 video_stream_encoder_->OnBitrateUpdated(
2827 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2828 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002829 DataRate::bps(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0, 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002830
2831 // Trigger adapt up. Higher resolution should be requested.
2832 video_stream_encoder_->TriggerQualityHigh();
2833 VerifyFpsMaxResolutionMax(source.sink_wants());
2834
2835 video_stream_encoder_->Stop();
2836}
2837
2838TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2839 fake_encoder_.SetResolutionBitrateLimits(
2840 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2841
2842 // Set bitrate equal to min bitrate of 540p.
2843 video_stream_encoder_->OnBitrateUpdated(
2844 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2845 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps),
Ying Wang9b881ab2020-02-07 14:29:32 +01002846 DataRate::bps(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0, 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002847
2848 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2849 AdaptingFrameForwarder source;
2850 source.set_adaptation_enabled(true);
2851 video_stream_encoder_->SetSource(
2852 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2853
2854 // Insert 720p frame. It should be dropped and lower resolution should be
2855 // requested.
2856 int64_t timestamp_ms = kFrameIntervalMs;
2857 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2858 ExpectDroppedFrame();
2859 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2860
2861 // Insert 720p frame. It should be downscaled and encoded.
2862 timestamp_ms += kFrameIntervalMs;
2863 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2864 WaitForEncodedFrame(960, 540);
2865
2866 video_stream_encoder_->Stop();
2867}
2868
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002869class BalancedDegradationTest : public VideoStreamEncoderTest {
2870 protected:
2871 void SetupTest() {
2872 // Reset encoder for field trials to take effect.
2873 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02002874 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002875
2876 // Enable BALANCED preference.
2877 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02002878 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
2879 }
2880
2881 void OnBitrateUpdated(int bitrate_bps) {
Ying Wang9b881ab2020-02-07 14:29:32 +01002882 video_stream_encoder_->OnBitrateUpdated(
2883 DataRate::bps(bitrate_bps), DataRate::bps(bitrate_bps),
2884 DataRate::bps(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002885 }
2886
Åsa Persson45b176f2019-09-30 11:19:05 +02002887 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002888 timestamp_ms_ += kFrameIntervalMs;
2889 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02002890 }
2891
2892 void InsertFrameAndWaitForEncoded() {
2893 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002894 sink_.WaitForEncodedFrame(timestamp_ms_);
2895 }
2896
2897 const int kWidth = 640; // pixels:640x360=230400
2898 const int kHeight = 360;
2899 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
2900 int64_t timestamp_ms_ = 0;
2901 AdaptingFrameForwarder source_;
2902};
2903
2904TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
2905 test::ScopedFieldTrials field_trials(
2906 "WebRTC-Video-BalancedDegradationSettings/"
2907 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
2908 SetupTest();
2909
2910 // Force input frame rate.
2911 const int kInputFps = 24;
2912 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2913 stats.input_frame_rate = kInputFps;
2914 stats_proxy_->SetMockStats(stats);
2915
Åsa Persson45b176f2019-09-30 11:19:05 +02002916 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002917 VerifyFpsMaxResolutionMax(source_.sink_wants());
2918
2919 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
2920 // Fps diff (input-requested:0) < threshold, expect AdaptDown to return false.
2921 video_stream_encoder_->TriggerQualityLowExpectFalse();
2922 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
2923
2924 video_stream_encoder_->Stop();
2925}
2926
2927TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
2928 test::ScopedFieldTrials field_trials(
2929 "WebRTC-Video-BalancedDegradationSettings/"
2930 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
2931 SetupTest();
2932
2933 // Force input frame rate.
2934 const int kInputFps = 25;
2935 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2936 stats.input_frame_rate = kInputFps;
2937 stats_proxy_->SetMockStats(stats);
2938
Åsa Persson45b176f2019-09-30 11:19:05 +02002939 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002940 VerifyFpsMaxResolutionMax(source_.sink_wants());
2941
2942 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
2943 // Fps diff (input-requested:1) == threshold, expect AdaptDown to return true.
2944 video_stream_encoder_->TriggerQualityLow();
2945 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
2946
2947 video_stream_encoder_->Stop();
2948}
2949
2950TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
2951 test::ScopedFieldTrials field_trials(
2952 "WebRTC-Video-BalancedDegradationSettings/"
2953 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
2954 SetupTest();
2955
2956 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
2957
Åsa Persson45b176f2019-09-30 11:19:05 +02002958 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002959 VerifyFpsMaxResolutionMax(source_.sink_wants());
2960
2961 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
2962 video_stream_encoder_->TriggerQualityLow();
2963 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
2964
2965 video_stream_encoder_->Stop();
2966}
2967
Åsa Perssonccfb3402019-09-25 15:13:04 +02002968TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002969 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02002970 "WebRTC-Video-BalancedDegradationSettings/"
2971 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02002972 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02002973
Åsa Persson1b247f12019-08-14 17:26:39 +02002974 const int kMinBitrateBps = 425000;
2975 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02002976 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02002977
Åsa Persson45b176f2019-09-30 11:19:05 +02002978 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02002979 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002980 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2981
2982 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
2983 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02002984 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02002985 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02002986 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2987
2988 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
2989 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02002990 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02002991 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02002992 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2993
Åsa Persson30ab0152019-08-27 12:22:33 +02002994 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
2995 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02002996 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02002997 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
2998 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02002999 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3000
3001 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003002 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003003 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003004 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003005
Åsa Persson30ab0152019-08-27 12:22:33 +02003006 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003007 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003008 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003009 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003010 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003011 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3012
3013 video_stream_encoder_->Stop();
3014}
3015
Åsa Perssonccfb3402019-09-25 15:13:04 +02003016TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003017 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3018 test::ScopedFieldTrials field_trials(
3019 "WebRTC-Video-BalancedDegradationSettings/"
3020 "pixels:57600|129600|230400,fps:7|24|24/");
3021 SetupTest();
3022 OnBitrateUpdated(kLowTargetBitrateBps);
3023
3024 VerifyNoLimitation(source_.sink_wants());
3025
3026 // Insert frame, expect scaled down:
3027 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3028 InsertFrame();
3029 EXPECT_FALSE(WaitForFrame(1000));
3030 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3031 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3032
3033 // Insert frame, expect scaled down:
3034 // resolution (320x180@24fps).
3035 InsertFrame();
3036 EXPECT_FALSE(WaitForFrame(1000));
3037 EXPECT_LT(source_.sink_wants().max_pixel_count,
3038 source_.last_wants().max_pixel_count);
3039 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3040
3041 // Frame should not be dropped (min pixels per frame reached).
3042 InsertFrameAndWaitForEncoded();
3043
3044 video_stream_encoder_->Stop();
3045}
3046
3047TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003048 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003049 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003050 "WebRTC-Video-BalancedDegradationSettings/"
3051 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003052 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003053
Åsa Persson30ab0152019-08-27 12:22:33 +02003054 const int kResolutionMinBitrateBps = 435000;
3055 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003056 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003057
Åsa Persson45b176f2019-09-30 11:19:05 +02003058 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003059 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003060 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3061
3062 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3063 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003064 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003065 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003066 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3067
3068 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3069 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003070 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003071 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003072 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3073
3074 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3075 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003076 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003077 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003078 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3079
Åsa Persson30ab0152019-08-27 12:22:33 +02003080 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3081 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003082 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003083 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003084 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3085
3086 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3087 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003088 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003089 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3090
3091 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003092 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003093 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003094 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003095 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003096 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3097
3098 video_stream_encoder_->Stop();
3099}
3100
Åsa Perssonccfb3402019-09-25 15:13:04 +02003101TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003102 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003103 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003104 "WebRTC-Video-BalancedDegradationSettings/"
3105 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003106 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003107
Åsa Persson30ab0152019-08-27 12:22:33 +02003108 const int kMinBitrateBps = 425000;
3109 const int kTooLowMinBitrateBps = 424000;
3110 const int kResolutionMinBitrateBps = 435000;
3111 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003112 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003113
Åsa Persson45b176f2019-09-30 11:19:05 +02003114 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003115 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003116 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3117
3118 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3119 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003120 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003121 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003122 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3123
3124 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3125 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003126 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003127 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003128 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3129
3130 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3131 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003132 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003133 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003134 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3135
3136 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3137 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003138 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003139 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3140
3141 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003142 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003143 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003144 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003145 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003146 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3147
3148 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003149 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003150 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003151 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003152 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3153
3154 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003155 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003156 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003157 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003158 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003159 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3160
Åsa Persson1b247f12019-08-14 17:26:39 +02003161 video_stream_encoder_->Stop();
3162}
3163
mflodmancc3d4422017-08-03 08:27:51 -07003164TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003165 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3166 const int kWidth = 1280;
3167 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02003168 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003169 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003170 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003171
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003172 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003173 AdaptingFrameForwarder source;
3174 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003175 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003176 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003177
Åsa Persson8c1bf952018-09-13 10:42:19 +02003178 int64_t timestamp_ms = kFrameIntervalMs;
3179 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003180 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003181 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003182 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3183 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3184 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3185 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3186
3187 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003188 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003189 timestamp_ms += kFrameIntervalMs;
3190 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3191 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003192 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003193 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3195 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3196 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3197
3198 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003199 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003200 timestamp_ms += kFrameIntervalMs;
3201 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3202 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003203 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003204 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3205 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3206 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3207 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3208
Jonathan Yubc771b72017-12-08 17:04:29 -08003209 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003210 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003211 timestamp_ms += kFrameIntervalMs;
3212 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3213 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003214 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003215 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3216 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003217 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003218 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3219
Jonathan Yubc771b72017-12-08 17:04:29 -08003220 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003221 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003222 timestamp_ms += kFrameIntervalMs;
3223 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3224 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003225 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003226 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003227 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3228 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3229 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3230 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3231
Jonathan Yubc771b72017-12-08 17:04:29 -08003232 // Trigger quality adapt down, expect no change (min resolution reached).
3233 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003234 timestamp_ms += kFrameIntervalMs;
3235 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3236 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003237 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3238 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3239 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3240 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3241 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3242
3243 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003244 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003245 timestamp_ms += kFrameIntervalMs;
3246 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3247 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003248 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003249 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3250 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3251 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3252 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3253
3254 // Trigger cpu adapt up, expect upscaled resolution (640x360).
3255 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003256 timestamp_ms += kFrameIntervalMs;
3257 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3258 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003259 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3260 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3261 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3262 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3263 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3264
3265 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3266 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003267 timestamp_ms += kFrameIntervalMs;
3268 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3269 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003270 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003271 last_wants = source.sink_wants();
3272 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3273 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003274 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003275 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3276
3277 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003278 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003279 timestamp_ms += kFrameIntervalMs;
3280 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3281 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003282 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003283 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3284 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003285 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003286 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3287
3288 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003289 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003290 timestamp_ms += kFrameIntervalMs;
3291 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003292 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003293 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003294 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003295 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003297 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003298 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003299
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003301}
3302
mflodmancc3d4422017-08-03 08:27:51 -07003303TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003304 const int kWidth = 640;
3305 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003306
Erik Språng4c6ca302019-04-08 15:14:01 +02003307 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003308 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003309 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003310
perkj803d97f2016-11-01 11:45:46 -07003311 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003312 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003313 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003314 }
3315
mflodmancc3d4422017-08-03 08:27:51 -07003316 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003317 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003318 video_source_.IncomingCapturedFrame(CreateFrame(
3319 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003320 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003321 }
3322
mflodmancc3d4422017-08-03 08:27:51 -07003323 video_stream_encoder_->Stop();
3324 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003325 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003326
Ying Wangef3998f2019-12-09 13:06:53 +01003327 EXPECT_METRIC_EQ(
3328 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3329 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003330 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3331}
3332
mflodmancc3d4422017-08-03 08:27:51 -07003333TEST_F(VideoStreamEncoderTest,
3334 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003335 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003336 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003337 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003338 const int kWidth = 640;
3339 const int kHeight = 360;
3340
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003341 video_stream_encoder_->SetSource(&video_source_,
3342 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003343
3344 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3345 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003346 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003347 }
3348
mflodmancc3d4422017-08-03 08:27:51 -07003349 video_stream_encoder_->Stop();
3350 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003351 stats_proxy_.reset();
3352
3353 EXPECT_EQ(0,
3354 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3355}
3356
mflodmancc3d4422017-08-03 08:27:51 -07003357TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003358 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003359 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003360
3361 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003362 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003363 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003364 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3365 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003366
sprang57c2fff2017-01-16 06:24:02 -08003367 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003368 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003369 video_stream_encoder_->OnBitrateUpdated(
3370 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003371 DataRate::bps(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003372
sprang57c2fff2017-01-16 06:24:02 -08003373 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003374 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3375 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003376 VideoBitrateAllocation bitrate_allocation =
3377 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003378 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003379 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003380 // TODO(srte): The use of millisecs here looks like an error, but the tests
3381 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003382 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003383
3384 // Not called on second frame.
3385 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3386 .Times(0);
3387 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003388 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3389 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003390 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003391
3392 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003393 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3394 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003395 const int64_t start_time_ms = rtc::TimeMillis();
3396 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3397 video_source_.IncomingCapturedFrame(
3398 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3399 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003400 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003401 }
3402
3403 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003404 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003405
mflodmancc3d4422017-08-03 08:27:51 -07003406 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003407}
3408
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003409TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3410 // 2 TLs configured, temporal layers supported by encoder.
3411 const int kNumTemporalLayers = 2;
3412 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3413 fake_encoder_.SetTemporalLayersSupported(0, true);
3414
3415 // Bitrate allocated across temporal layers.
3416 const int kTl0Bps = kTargetBitrateBps *
3417 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003418 kNumTemporalLayers, /*temporal_id*/ 0,
3419 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003420 const int kTl1Bps = kTargetBitrateBps *
3421 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003422 kNumTemporalLayers, /*temporal_id*/ 1,
3423 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003424 VideoBitrateAllocation expected_bitrate;
3425 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3426 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3427
3428 VerifyAllocatedBitrate(expected_bitrate);
3429 video_stream_encoder_->Stop();
3430}
3431
3432TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3433 // 2 TLs configured, temporal layers not supported by encoder.
3434 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3435 fake_encoder_.SetTemporalLayersSupported(0, false);
3436
3437 // Temporal layers not supported by the encoder.
3438 // Total bitrate should be at ti:0.
3439 VideoBitrateAllocation expected_bitrate;
3440 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3441
3442 VerifyAllocatedBitrate(expected_bitrate);
3443 video_stream_encoder_->Stop();
3444}
3445
3446TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3447 // 2 TLs configured, temporal layers only supported for first stream.
3448 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3449 fake_encoder_.SetTemporalLayersSupported(0, true);
3450 fake_encoder_.SetTemporalLayersSupported(1, false);
3451
3452 const int kS0Bps = 150000;
3453 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003454 kS0Bps *
3455 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3456 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003457 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003458 kS0Bps *
3459 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3460 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003461 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3462 // Temporal layers not supported by si:1.
3463 VideoBitrateAllocation expected_bitrate;
3464 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3465 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3466 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3467
3468 VerifyAllocatedBitrate(expected_bitrate);
3469 video_stream_encoder_->Stop();
3470}
3471
Niels Möller7dc26b72017-12-06 10:27:48 +01003472TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3473 const int kFrameWidth = 1280;
3474 const int kFrameHeight = 720;
3475 const int kFramerate = 24;
3476
Erik Språng4c6ca302019-04-08 15:14:01 +02003477 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003478 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003479 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003480 test::FrameForwarder source;
3481 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003482 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003483
3484 // Insert a single frame, triggering initial configuration.
3485 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3486 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3487
3488 EXPECT_EQ(
3489 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3490 kDefaultFramerate);
3491
3492 // Trigger reconfigure encoder (without resetting the entire instance).
3493 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003494 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003495 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3496 video_encoder_config.number_of_streams = 1;
3497 video_encoder_config.video_stream_factory =
3498 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3499 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003500 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003501 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3502
3503 // Detector should be updated with fps limit from codec config.
3504 EXPECT_EQ(
3505 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3506 kFramerate);
3507
3508 // Trigger overuse, max framerate should be reduced.
3509 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3510 stats.input_frame_rate = kFramerate;
3511 stats_proxy_->SetMockStats(stats);
3512 video_stream_encoder_->TriggerCpuOveruse();
3513 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3514 int adapted_framerate =
3515 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3516 EXPECT_LT(adapted_framerate, kFramerate);
3517
3518 // Trigger underuse, max framerate should go back to codec configured fps.
3519 // Set extra low fps, to make sure it's actually reset, not just incremented.
3520 stats = stats_proxy_->GetStats();
3521 stats.input_frame_rate = adapted_framerate / 2;
3522 stats_proxy_->SetMockStats(stats);
3523 video_stream_encoder_->TriggerCpuNormalUsage();
3524 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3525 EXPECT_EQ(
3526 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3527 kFramerate);
3528
3529 video_stream_encoder_->Stop();
3530}
3531
3532TEST_F(VideoStreamEncoderTest,
3533 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3534 const int kFrameWidth = 1280;
3535 const int kFrameHeight = 720;
3536 const int kLowFramerate = 15;
3537 const int kHighFramerate = 25;
3538
Erik Språng4c6ca302019-04-08 15:14:01 +02003539 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003540 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003541 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003542 test::FrameForwarder source;
3543 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003544 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003545
3546 // Trigger initial configuration.
3547 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003548 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003549 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3550 video_encoder_config.number_of_streams = 1;
3551 video_encoder_config.video_stream_factory =
3552 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3553 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3554 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003555 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003556 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3557
3558 EXPECT_EQ(
3559 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3560 kLowFramerate);
3561
3562 // Trigger overuse, max framerate should be reduced.
3563 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3564 stats.input_frame_rate = kLowFramerate;
3565 stats_proxy_->SetMockStats(stats);
3566 video_stream_encoder_->TriggerCpuOveruse();
3567 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3568 int adapted_framerate =
3569 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3570 EXPECT_LT(adapted_framerate, kLowFramerate);
3571
3572 // Reconfigure the encoder with a new (higher max framerate), max fps should
3573 // still respect the adaptation.
3574 video_encoder_config.video_stream_factory =
3575 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3576 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3577 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003578 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003579 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3580
3581 EXPECT_EQ(
3582 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3583 adapted_framerate);
3584
3585 // Trigger underuse, max framerate should go back to codec configured fps.
3586 stats = stats_proxy_->GetStats();
3587 stats.input_frame_rate = adapted_framerate;
3588 stats_proxy_->SetMockStats(stats);
3589 video_stream_encoder_->TriggerCpuNormalUsage();
3590 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3591 EXPECT_EQ(
3592 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3593 kHighFramerate);
3594
3595 video_stream_encoder_->Stop();
3596}
3597
mflodmancc3d4422017-08-03 08:27:51 -07003598TEST_F(VideoStreamEncoderTest,
3599 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003600 const int kFrameWidth = 1280;
3601 const int kFrameHeight = 720;
3602 const int kFramerate = 24;
3603
Erik Språng4c6ca302019-04-08 15:14:01 +02003604 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003605 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003606 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003607 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003608 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003609 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003610
3611 // Trigger initial configuration.
3612 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003613 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003614 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3615 video_encoder_config.number_of_streams = 1;
3616 video_encoder_config.video_stream_factory =
3617 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3618 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003619 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003620 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003621 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003622
Niels Möller7dc26b72017-12-06 10:27:48 +01003623 EXPECT_EQ(
3624 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3625 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003626
3627 // Trigger overuse, max framerate should be reduced.
3628 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3629 stats.input_frame_rate = kFramerate;
3630 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003631 video_stream_encoder_->TriggerCpuOveruse();
3632 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003633 int adapted_framerate =
3634 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003635 EXPECT_LT(adapted_framerate, kFramerate);
3636
3637 // Change degradation preference to not enable framerate scaling. Target
3638 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003639 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003640 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003641 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003642 EXPECT_EQ(
3643 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3644 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003645
mflodmancc3d4422017-08-03 08:27:51 -07003646 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003647}
3648
mflodmancc3d4422017-08-03 08:27:51 -07003649TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003650 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003651 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003652 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003653 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003654 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003655 const int kWidth = 640;
3656 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003657
asaperssonfab67072017-04-04 05:51:49 -07003658 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003659
3660 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003661 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003662
3663 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003664 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003665
sprangc5d62e22017-04-02 23:53:04 -07003666 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003667
asaperssonfab67072017-04-04 05:51:49 -07003668 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003669 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003670 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003671
3672 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003673 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003674
sprangc5d62e22017-04-02 23:53:04 -07003675 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003676
mflodmancc3d4422017-08-03 08:27:51 -07003677 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003678}
3679
mflodmancc3d4422017-08-03 08:27:51 -07003680TEST_F(VideoStreamEncoderTest,
3681 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003682 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003683 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02003684 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003685 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003686 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003687 const int kWidth = 640;
3688 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003689
3690 // We expect the n initial frames to get dropped.
3691 int i;
3692 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003693 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003694 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003695 }
3696 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003697 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003698 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003699
3700 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003701 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003702
mflodmancc3d4422017-08-03 08:27:51 -07003703 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003704}
3705
mflodmancc3d4422017-08-03 08:27:51 -07003706TEST_F(VideoStreamEncoderTest,
3707 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003708 const int kWidth = 640;
3709 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003710 video_stream_encoder_->OnBitrateUpdated(
3711 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003712 DataRate::bps(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003713
3714 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003715 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003716 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003717
asaperssonfab67072017-04-04 05:51:49 -07003718 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003719 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003720 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003721
mflodmancc3d4422017-08-03 08:27:51 -07003722 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003723}
3724
mflodmancc3d4422017-08-03 08:27:51 -07003725TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003726 const int kWidth = 640;
3727 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003728 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003729
3730 VideoEncoderConfig video_encoder_config;
3731 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3732 // Make format different, to force recreation of encoder.
3733 video_encoder_config.video_format.parameters["foo"] = "foo";
3734 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003735 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003736 video_stream_encoder_->OnBitrateUpdated(
3737 DataRate::bps(kLowTargetBitrateBps), DataRate::bps(kLowTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003738 DataRate::bps(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003739
kthelgasonb83797b2017-02-14 11:57:25 -08003740 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003741 video_stream_encoder_->SetSource(&video_source_,
3742 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003743
asaperssonfab67072017-04-04 05:51:49 -07003744 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003745 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003746 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003747
mflodmancc3d4422017-08-03 08:27:51 -07003748 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003749 fake_encoder_.SetQualityScaling(true);
3750}
3751
Åsa Persson139f4dc2019-08-02 09:29:58 +02003752TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3753 webrtc::test::ScopedFieldTrials field_trials(
3754 "WebRTC-Video-QualityScalerSettings/"
3755 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3756 // Reset encoder for field trials to take effect.
3757 ConfigureEncoder(video_encoder_config_.Copy());
3758 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3759 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3760 const int kWidth = 640;
3761 const int kHeight = 360;
3762
3763 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003764 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003765 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003766 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3767 // Frame should not be dropped.
3768 WaitForEncodedFrame(1);
3769
3770 video_stream_encoder_->OnBitrateUpdated(
3771 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003772 DataRate::bps(kNotTooLowBitrateForFrameSizeBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003773 DataRate::bps(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003774 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3775 // Frame should not be dropped.
3776 WaitForEncodedFrame(2);
3777
3778 video_stream_encoder_->OnBitrateUpdated(
3779 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Florent Castellia8336d32019-09-09 13:36:55 +02003780 DataRate::bps(kTooLowBitrateForFrameSizeBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003781 DataRate::bps(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003782 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3783 // Expect to drop this frame, the wait should time out.
3784 ExpectDroppedFrame();
3785
3786 // Expect the sink_wants to specify a scaled frame.
3787 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3788 video_stream_encoder_->Stop();
3789}
3790
Åsa Perssone644a032019-11-08 15:56:00 +01003791TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3792 webrtc::test::ScopedFieldTrials field_trials(
3793 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3794
3795 // Reset encoder for field trials to take effect.
3796 VideoEncoderConfig config = video_encoder_config_.Copy();
3797 config.max_bitrate_bps = kTargetBitrateBps;
3798 ConfigureEncoder(std::move(config));
3799 fake_encoder_.SetQp(kQpLow);
3800
3801 // Enable MAINTAIN_FRAMERATE preference.
3802 AdaptingFrameForwarder source;
3803 source.set_adaptation_enabled(true);
3804 video_stream_encoder_->SetSource(&source,
3805 DegradationPreference::MAINTAIN_FRAMERATE);
3806
3807 // Start at low bitrate.
3808 const int kLowBitrateBps = 200000;
Ying Wang9b881ab2020-02-07 14:29:32 +01003809 video_stream_encoder_->OnBitrateUpdated(
3810 DataRate::bps(kLowBitrateBps), DataRate::bps(kLowBitrateBps),
3811 DataRate::bps(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003812
3813 // Expect first frame to be dropped and resolution to be limited.
3814 const int kWidth = 1280;
3815 const int kHeight = 720;
3816 const int64_t kFrameIntervalMs = 100;
3817 int64_t timestamp_ms = kFrameIntervalMs;
3818 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3819 ExpectDroppedFrame();
3820 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3821
3822 // Increase bitrate to encoder max.
3823 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(config.max_bitrate_bps),
3824 DataRate::bps(config.max_bitrate_bps),
3825 DataRate::bps(config.max_bitrate_bps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003826 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003827
3828 // Insert frames and advance |min_duration_ms|.
3829 for (size_t i = 1; i <= 10; i++) {
3830 timestamp_ms += kFrameIntervalMs;
3831 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3832 WaitForEncodedFrame(timestamp_ms);
3833 }
3834 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3835 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3836
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003837 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01003838
3839 // Insert frame should trigger high BW and release quality limitation.
3840 timestamp_ms += kFrameIntervalMs;
3841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3842 WaitForEncodedFrame(timestamp_ms);
3843 VerifyFpsMaxResolutionMax(source.sink_wants());
3844
3845 // Frame should not be adapted.
3846 timestamp_ms += kFrameIntervalMs;
3847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3848 WaitForEncodedFrame(kWidth, kHeight);
3849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3850
3851 video_stream_encoder_->Stop();
3852}
3853
mflodmancc3d4422017-08-03 08:27:51 -07003854TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003855 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3856 const int kTooSmallWidth = 10;
3857 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02003858 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003859 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003860 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003861
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003862 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003863 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003864 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003865 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003866 VerifyNoLimitation(source.sink_wants());
3867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3868
3869 // Trigger adapt down, too small frame, expect no change.
3870 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003871 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003872 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003873 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003874 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3875 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3876
mflodmancc3d4422017-08-03 08:27:51 -07003877 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003878}
3879
mflodmancc3d4422017-08-03 08:27:51 -07003880TEST_F(VideoStreamEncoderTest,
3881 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003882 const int kTooSmallWidth = 10;
3883 const int kTooSmallHeight = 10;
3884 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02003885 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003886 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003887 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003888
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003889 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003890 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003891 video_stream_encoder_->SetSource(&source,
3892 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003893 VerifyNoLimitation(source.sink_wants());
3894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3895 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3896
3897 // Trigger adapt down, expect limited framerate.
3898 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003899 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003900 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003901 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3902 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3903 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3904 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3905
3906 // Trigger adapt down, too small frame, expect no change.
3907 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003908 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003909 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003910 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
3911 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3912 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3913 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3914
mflodmancc3d4422017-08-03 08:27:51 -07003915 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003916}
3917
mflodmancc3d4422017-08-03 08:27:51 -07003918TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07003919 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02003920 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003921 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003922 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02003923 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07003924 const int kFrameWidth = 1280;
3925 const int kFrameHeight = 720;
3926 video_source_.IncomingCapturedFrame(
3927 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003928 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07003929 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003930}
3931
sprangb1ca0732017-02-01 08:38:12 -08003932// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07003933TEST_F(VideoStreamEncoderTest,
3934 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003935 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003936 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003937 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08003938
3939 const int kFrameWidth = 1280;
3940 const int kFrameHeight = 720;
3941 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07003942 // requested by
3943 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08003944 video_source_.set_adaptation_enabled(true);
3945
3946 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003947 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003948 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003949
3950 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07003951 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08003952 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003953 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003954 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08003955
asaperssonfab67072017-04-04 05:51:49 -07003956 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003957 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08003958 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003960 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08003961
mflodmancc3d4422017-08-03 08:27:51 -07003962 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08003963}
sprangfe627f32017-03-29 08:24:59 -07003964
mflodmancc3d4422017-08-03 08:27:51 -07003965TEST_F(VideoStreamEncoderTest,
3966 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07003967 const int kFrameWidth = 1280;
3968 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07003969
Erik Språng4c6ca302019-04-08 15:14:01 +02003970 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02003971 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01003972 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07003973 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003974 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003975 video_source_.set_adaptation_enabled(true);
3976
sprang4847ae62017-06-27 07:06:52 -07003977 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07003978
3979 video_source_.IncomingCapturedFrame(
3980 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003981 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003982
3983 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07003984 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003985
3986 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07003987 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003988 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003989 video_source_.IncomingCapturedFrame(
3990 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07003991 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07003992 }
3993
3994 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07003995 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003996 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07003997 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07003998 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07003999 video_source_.IncomingCapturedFrame(
4000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004001 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004002 ++num_frames_dropped;
4003 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004004 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004005 }
4006 }
4007
sprang4847ae62017-06-27 07:06:52 -07004008 // Add some slack to account for frames dropped by the frame dropper.
4009 const int kErrorMargin = 1;
4010 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004011 kErrorMargin);
4012
4013 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004014 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004015 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004016 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004017 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004018 video_source_.IncomingCapturedFrame(
4019 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004020 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004021 ++num_frames_dropped;
4022 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004023 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004024 }
4025 }
sprang4847ae62017-06-27 07:06:52 -07004026 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004027 kErrorMargin);
4028
4029 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07004030 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07004031 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004032 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004033 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004034 video_source_.IncomingCapturedFrame(
4035 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004036 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004037 ++num_frames_dropped;
4038 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004039 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004040 }
4041 }
sprang4847ae62017-06-27 07:06:52 -07004042 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004043 kErrorMargin);
4044
4045 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07004046 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07004047 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004048 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004049 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004050 video_source_.IncomingCapturedFrame(
4051 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004052 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004053 ++num_frames_dropped;
4054 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004055 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004056 }
4057 }
4058 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4059
mflodmancc3d4422017-08-03 08:27:51 -07004060 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004061}
4062
mflodmancc3d4422017-08-03 08:27:51 -07004063TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004064 const int kFramerateFps = 5;
4065 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004066 const int kFrameWidth = 1280;
4067 const int kFrameHeight = 720;
4068
sprang4847ae62017-06-27 07:06:52 -07004069 // Reconfigure encoder with two temporal layers and screensharing, which will
4070 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004071 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004072
Erik Språng4c6ca302019-04-08 15:14:01 +02004073 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004074 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004075 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004076 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004077 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004078 video_source_.set_adaptation_enabled(true);
4079
sprang4847ae62017-06-27 07:06:52 -07004080 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004081
4082 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004083 rtc::VideoSinkWants last_wants;
4084 do {
4085 last_wants = video_source_.sink_wants();
4086
sprangc5d62e22017-04-02 23:53:04 -07004087 // Insert frames to get a new fps estimate...
4088 for (int j = 0; j < kFramerateFps; ++j) {
4089 video_source_.IncomingCapturedFrame(
4090 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004091 if (video_source_.last_sent_width()) {
4092 sink_.WaitForEncodedFrame(timestamp_ms);
4093 }
sprangc5d62e22017-04-02 23:53:04 -07004094 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004095 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004096 }
4097 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004098 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004099 } while (video_source_.sink_wants().max_framerate_fps <
4100 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004101
Jonathan Yubc771b72017-12-08 17:04:29 -08004102 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004103
mflodmancc3d4422017-08-03 08:27:51 -07004104 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004105}
asaperssonf7e294d2017-06-13 23:25:22 -07004106
mflodmancc3d4422017-08-03 08:27:51 -07004107TEST_F(VideoStreamEncoderTest,
4108 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004109 const int kWidth = 1280;
4110 const int kHeight = 720;
4111 const int64_t kFrameIntervalMs = 150;
4112 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004113 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004114 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004115 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004116
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004117 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004118 AdaptingFrameForwarder source;
4119 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004120 video_stream_encoder_->SetSource(&source,
4121 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004122 timestamp_ms += kFrameIntervalMs;
4123 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004124 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004125 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004126 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4127 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4128 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4129
4130 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004131 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004132 timestamp_ms += kFrameIntervalMs;
4133 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004134 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004135 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4136 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4137 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4138 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4139
4140 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004141 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004142 timestamp_ms += kFrameIntervalMs;
4143 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004144 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004145 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4146 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4147 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4148 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4149
4150 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004151 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004152 timestamp_ms += kFrameIntervalMs;
4153 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004154 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004155 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4156 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4157 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4158 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4159
4160 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004161 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004162 timestamp_ms += kFrameIntervalMs;
4163 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004164 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004165 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4166 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4168 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4169
4170 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004171 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004172 timestamp_ms += kFrameIntervalMs;
4173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004174 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004175 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4176 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4177 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4178 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4179
4180 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004181 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004182 timestamp_ms += kFrameIntervalMs;
4183 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004184 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004185 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4186 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4187 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4188 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4189
4190 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004191 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004192 timestamp_ms += kFrameIntervalMs;
4193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004194 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004195 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4196 rtc::VideoSinkWants last_wants = source.sink_wants();
4197 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4198 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4199 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4200
4201 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004202 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004203 timestamp_ms += kFrameIntervalMs;
4204 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004205 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004206 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4207 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4208 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4209 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4210
4211 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004212 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004213 timestamp_ms += kFrameIntervalMs;
4214 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004215 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004216 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4217 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4218 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4219 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4220
4221 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004222 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004223 timestamp_ms += kFrameIntervalMs;
4224 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004225 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004226 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4227 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4228 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4229 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4230
4231 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004232 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004233 timestamp_ms += kFrameIntervalMs;
4234 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004235 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004236 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4237 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4238 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4239 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4240
4241 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004242 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004243 timestamp_ms += kFrameIntervalMs;
4244 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004245 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004246 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4247 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4248 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4249 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4250
4251 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004252 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004253 timestamp_ms += kFrameIntervalMs;
4254 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004255 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004256 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4257 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4258 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4259 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4260
4261 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004262 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004263 timestamp_ms += kFrameIntervalMs;
4264 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004265 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004266 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4267 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4269 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4270
Åsa Persson30ab0152019-08-27 12:22:33 +02004271 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004272 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004273 timestamp_ms += kFrameIntervalMs;
4274 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004275 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004276 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004277 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004278 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4279 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4280 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4281
4282 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004283 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004284 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004285 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4286
mflodmancc3d4422017-08-03 08:27:51 -07004287 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004288}
4289
mflodmancc3d4422017-08-03 08:27:51 -07004290TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004291 const int kWidth = 1280;
4292 const int kHeight = 720;
4293 const int64_t kFrameIntervalMs = 150;
4294 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004295 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004296 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004297 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004298
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004299 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004300 AdaptingFrameForwarder source;
4301 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004302 video_stream_encoder_->SetSource(&source,
4303 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004304 timestamp_ms += kFrameIntervalMs;
4305 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004306 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004307 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4309 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4310 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4311 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4312 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4313 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4314
4315 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004316 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004317 timestamp_ms += kFrameIntervalMs;
4318 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004319 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004320 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4321 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4322 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4323 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4324 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4325 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4326 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4327
4328 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004329 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004330 timestamp_ms += kFrameIntervalMs;
4331 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004332 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004333 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4334 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4335 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4336 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4337 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4338 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4339 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4340
4341 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004342 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004343 timestamp_ms += kFrameIntervalMs;
4344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004345 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004346 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4347 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4349 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4350 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4351 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4352 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4353
4354 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004355 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004356 timestamp_ms += kFrameIntervalMs;
4357 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004358 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004359 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4360 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4361 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4362 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4363 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4364 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4365 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4366
4367 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004368 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004369 timestamp_ms += kFrameIntervalMs;
4370 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004371 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004372 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4373 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4374 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4375 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4376 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4377 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4378 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4379
4380 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004381 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004382 timestamp_ms += kFrameIntervalMs;
4383 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004384 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004385 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004386 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004387 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4388 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4389 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4390 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4391 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4392 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4393
4394 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004395 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004396 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004397 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4398 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4399
mflodmancc3d4422017-08-03 08:27:51 -07004400 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004401}
4402
mflodmancc3d4422017-08-03 08:27:51 -07004403TEST_F(VideoStreamEncoderTest,
4404 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004405 const int kWidth = 640;
4406 const int kHeight = 360;
4407 const int kFpsLimit = 15;
4408 const int64_t kFrameIntervalMs = 150;
4409 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004410 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004411 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004412 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004413
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004414 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004415 AdaptingFrameForwarder source;
4416 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004417 video_stream_encoder_->SetSource(&source,
4418 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004419 timestamp_ms += kFrameIntervalMs;
4420 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004421 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004422 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4424 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4425 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4426 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4427 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4428 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4429
4430 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004431 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004432 timestamp_ms += kFrameIntervalMs;
4433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004434 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004435 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4436 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4438 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4439 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4440 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4441 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4442
4443 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004444 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004445 timestamp_ms += kFrameIntervalMs;
4446 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004447 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004448 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4449 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4450 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4451 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4452 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4453 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4454 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4455
4456 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004457 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004458 timestamp_ms += kFrameIntervalMs;
4459 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004460 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004461 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4462 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4463 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4464 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4465 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4466 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4467 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4468
4469 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004470 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004471 timestamp_ms += kFrameIntervalMs;
4472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004473 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004474 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004475 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4476 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4477 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4478 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4479 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4480 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4481
4482 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004483 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004484 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004485 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4486 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4487
mflodmancc3d4422017-08-03 08:27:51 -07004488 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004489}
4490
mflodmancc3d4422017-08-03 08:27:51 -07004491TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004492 const int kFrameWidth = 1920;
4493 const int kFrameHeight = 1080;
4494 // 3/4 of 1920.
4495 const int kAdaptedFrameWidth = 1440;
4496 // 3/4 of 1080 rounded down to multiple of 4.
4497 const int kAdaptedFrameHeight = 808;
4498 const int kFramerate = 24;
4499
Erik Språng4c6ca302019-04-08 15:14:01 +02004500 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004501 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004502 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004503 // Trigger reconfigure encoder (without resetting the entire instance).
4504 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004505 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004506 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4507 video_encoder_config.number_of_streams = 1;
4508 video_encoder_config.video_stream_factory =
4509 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004510 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004511 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004512 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004513
4514 video_source_.set_adaptation_enabled(true);
4515
4516 video_source_.IncomingCapturedFrame(
4517 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004518 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004519
4520 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004521 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004522 video_source_.IncomingCapturedFrame(
4523 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004524 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004525
mflodmancc3d4422017-08-03 08:27:51 -07004526 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004527}
4528
mflodmancc3d4422017-08-03 08:27:51 -07004529TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004530 const int kFrameWidth = 1280;
4531 const int kFrameHeight = 720;
4532 const int kLowFps = 2;
4533 const int kHighFps = 30;
4534
Erik Språng4c6ca302019-04-08 15:14:01 +02004535 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004536 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004537 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004538
4539 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4540 max_framerate_ = kLowFps;
4541
4542 // Insert 2 seconds of 2fps video.
4543 for (int i = 0; i < kLowFps * 2; ++i) {
4544 video_source_.IncomingCapturedFrame(
4545 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4546 WaitForEncodedFrame(timestamp_ms);
4547 timestamp_ms += 1000 / kLowFps;
4548 }
4549
4550 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004551 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004552 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004553 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004554 video_source_.IncomingCapturedFrame(
4555 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4556 WaitForEncodedFrame(timestamp_ms);
4557 timestamp_ms += 1000 / kLowFps;
4558
4559 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4560
4561 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004562 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004563 const int kFrameIntervalMs = 1000 / kHighFps;
4564 max_framerate_ = kHighFps;
4565 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4566 video_source_.IncomingCapturedFrame(
4567 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4568 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4569 // be dropped if the encoder hans't been updated with the new higher target
4570 // framerate yet, causing it to overshoot the target bitrate and then
4571 // suffering the wrath of the media optimizer.
4572 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4573 timestamp_ms += kFrameIntervalMs;
4574 }
4575
4576 // Don expect correct measurement just yet, but it should be higher than
4577 // before.
4578 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4579
mflodmancc3d4422017-08-03 08:27:51 -07004580 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004581}
4582
mflodmancc3d4422017-08-03 08:27:51 -07004583TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004584 const int kFrameWidth = 1280;
4585 const int kFrameHeight = 720;
4586 const int kTargetBitrateBps = 1000000;
4587
4588 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004589 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004590 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004591 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004592 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004593 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004594
4595 // Insert a first video frame, causes another bitrate update.
4596 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4597 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4598 video_source_.IncomingCapturedFrame(
4599 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4600 WaitForEncodedFrame(timestamp_ms);
4601
4602 // Next, simulate video suspension due to pacer queue overrun.
Florent Castellia8336d32019-09-09 13:36:55 +02004603 video_stream_encoder_->OnBitrateUpdated(DataRate::bps(0), DataRate::bps(0),
Ying Wang9b881ab2020-02-07 14:29:32 +01004604 DataRate::bps(0), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004605
4606 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004607 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004608 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004609
4610 // Bitrate observer should not be called.
4611 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4612 video_source_.IncomingCapturedFrame(
4613 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4614 ExpectDroppedFrame();
4615
mflodmancc3d4422017-08-03 08:27:51 -07004616 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004617}
ilnik6b826ef2017-06-16 06:53:48 -07004618
Niels Möller4db138e2018-04-19 09:04:13 +02004619TEST_F(VideoStreamEncoderTest,
4620 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4621 const int kFrameWidth = 1280;
4622 const int kFrameHeight = 720;
4623 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004624 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004625 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004626 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004627 video_source_.IncomingCapturedFrame(
4628 CreateFrame(1, kFrameWidth, kFrameHeight));
4629 WaitForEncodedFrame(1);
4630 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4631 .low_encode_usage_threshold_percent,
4632 default_options.low_encode_usage_threshold_percent);
4633 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4634 .high_encode_usage_threshold_percent,
4635 default_options.high_encode_usage_threshold_percent);
4636 video_stream_encoder_->Stop();
4637}
4638
4639TEST_F(VideoStreamEncoderTest,
4640 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4641 const int kFrameWidth = 1280;
4642 const int kFrameHeight = 720;
4643 CpuOveruseOptions hardware_options;
4644 hardware_options.low_encode_usage_threshold_percent = 150;
4645 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004646 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004647
Erik Språng4c6ca302019-04-08 15:14:01 +02004648 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004649 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004650 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004651 video_source_.IncomingCapturedFrame(
4652 CreateFrame(1, kFrameWidth, kFrameHeight));
4653 WaitForEncodedFrame(1);
4654 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4655 .low_encode_usage_threshold_percent,
4656 hardware_options.low_encode_usage_threshold_percent);
4657 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4658 .high_encode_usage_threshold_percent,
4659 hardware_options.high_encode_usage_threshold_percent);
4660 video_stream_encoder_->Stop();
4661}
4662
Niels Möller6bb5ab92019-01-11 11:11:10 +01004663TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4664 const int kFrameWidth = 320;
4665 const int kFrameHeight = 240;
4666 const int kFps = 30;
4667 const int kTargetBitrateBps = 120000;
4668 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4669
Erik Språng4c6ca302019-04-08 15:14:01 +02004670 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004671 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004672 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004673
4674 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4675 max_framerate_ = kFps;
4676
4677 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4678 fake_encoder_.SimulateOvershoot(1.0);
4679 int num_dropped = 0;
4680 for (int i = 0; i < kNumFramesInRun; ++i) {
4681 video_source_.IncomingCapturedFrame(
4682 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4683 // Wait up to two frame durations for a frame to arrive.
4684 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4685 ++num_dropped;
4686 }
4687 timestamp_ms += 1000 / kFps;
4688 }
4689
Erik Språnga8d48ab2019-02-08 14:17:40 +01004690 // Framerate should be measured to be near the expected target rate.
4691 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4692
4693 // Frame drops should be within 5% of expected 0%.
4694 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004695
4696 // Make encoder produce frames at double the expected bitrate during 3 seconds
4697 // of video, verify number of drops. Rate needs to be slightly changed in
4698 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004699 double overshoot_factor = 2.0;
4700 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4701 // With bitrate adjuster, when need to overshoot even more to trigger
4702 // frame dropping.
4703 overshoot_factor *= 2;
4704 }
4705 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004706 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004707 DataRate::bps(kTargetBitrateBps + 1000),
Florent Castellia8336d32019-09-09 13:36:55 +02004708 DataRate::bps(kTargetBitrateBps + 1000),
Ying Wang9b881ab2020-02-07 14:29:32 +01004709 DataRate::bps(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004710 num_dropped = 0;
4711 for (int i = 0; i < kNumFramesInRun; ++i) {
4712 video_source_.IncomingCapturedFrame(
4713 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4714 // Wait up to two frame durations for a frame to arrive.
4715 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4716 ++num_dropped;
4717 }
4718 timestamp_ms += 1000 / kFps;
4719 }
4720
Erik Språng4c6ca302019-04-08 15:14:01 +02004721 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004722 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004723 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004724
4725 // Target framerate should be still be near the expected target, despite
4726 // the frame drops.
4727 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4728
4729 // Frame drops should be within 5% of expected 50%.
4730 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004731
4732 video_stream_encoder_->Stop();
4733}
4734
4735TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4736 const int kFrameWidth = 320;
4737 const int kFrameHeight = 240;
4738 const int kActualInputFps = 24;
4739 const int kTargetBitrateBps = 120000;
4740
4741 ASSERT_GT(max_framerate_, kActualInputFps);
4742
4743 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4744 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004745 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004746 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004747 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004748
4749 // Insert 3 seconds of video, with an input fps lower than configured max.
4750 for (int i = 0; i < kActualInputFps * 3; ++i) {
4751 video_source_.IncomingCapturedFrame(
4752 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4753 // Wait up to two frame durations for a frame to arrive.
4754 WaitForEncodedFrame(timestamp_ms);
4755 timestamp_ms += 1000 / kActualInputFps;
4756 }
4757
4758 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4759
4760 video_stream_encoder_->Stop();
4761}
4762
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004763TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4764 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004765 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004766 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004767 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004768
4769 fake_encoder_.BlockNextEncode();
4770 video_source_.IncomingCapturedFrame(
4771 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4772 WaitForEncodedFrame(1);
4773 // On the very first frame full update should be forced.
4774 rect = fake_encoder_.GetLastUpdateRect();
4775 EXPECT_EQ(rect.offset_x, 0);
4776 EXPECT_EQ(rect.offset_y, 0);
4777 EXPECT_EQ(rect.height, codec_height_);
4778 EXPECT_EQ(rect.width, codec_width_);
4779 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4780 // call to ContinueEncode.
4781 video_source_.IncomingCapturedFrame(
4782 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4783 ExpectDroppedFrame();
4784 video_source_.IncomingCapturedFrame(
4785 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4786 ExpectDroppedFrame();
4787 fake_encoder_.ContinueEncode();
4788 WaitForEncodedFrame(3);
4789 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4790 rect = fake_encoder_.GetLastUpdateRect();
4791 EXPECT_EQ(rect.offset_x, 1);
4792 EXPECT_EQ(rect.offset_y, 0);
4793 EXPECT_EQ(rect.width, 10);
4794 EXPECT_EQ(rect.height, 1);
4795
4796 video_source_.IncomingCapturedFrame(
4797 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4798 WaitForEncodedFrame(4);
4799 // Previous frame was encoded, so no accumulation should happen.
4800 rect = fake_encoder_.GetLastUpdateRect();
4801 EXPECT_EQ(rect.offset_x, 0);
4802 EXPECT_EQ(rect.offset_y, 0);
4803 EXPECT_EQ(rect.width, 1);
4804 EXPECT_EQ(rect.height, 1);
4805
4806 video_stream_encoder_->Stop();
4807}
4808
Erik Språngd7329ca2019-02-21 21:19:53 +01004809TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004810 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004811 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004812 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004813
4814 // First frame is always keyframe.
4815 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4816 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004817 EXPECT_THAT(
4818 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004819 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004820
4821 // Insert delta frame.
4822 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4823 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01004824 EXPECT_THAT(
4825 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004826 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004827
4828 // Request next frame be a key-frame.
4829 video_stream_encoder_->SendKeyFrame();
4830 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4831 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01004832 EXPECT_THAT(
4833 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004834 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004835
4836 video_stream_encoder_->Stop();
4837}
4838
4839TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4840 // Setup simulcast with three streams.
4841 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01004842 video_stream_encoder_->OnBitrateUpdated(
Erik Språng4c6ca302019-04-08 15:14:01 +02004843 DataRate::bps(kSimulcastTargetBitrateBps),
Florent Castellia8336d32019-09-09 13:36:55 +02004844 DataRate::bps(kSimulcastTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004845 DataRate::bps(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004846 // Wait for all three layers before triggering event.
4847 sink_.SetNumExpectedLayers(3);
4848
4849 // First frame is always keyframe.
4850 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4851 WaitForEncodedFrame(1);
4852 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004853 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4854 VideoFrameType::kVideoFrameKey,
4855 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004856
4857 // Insert delta frame.
4858 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4859 WaitForEncodedFrame(2);
4860 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004861 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4862 VideoFrameType::kVideoFrameDelta,
4863 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004864
4865 // Request next frame be a key-frame.
4866 // Only first stream is configured to produce key-frame.
4867 video_stream_encoder_->SendKeyFrame();
4868 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4869 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02004870
4871 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4872 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01004873 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004874 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02004875 VideoFrameType::kVideoFrameKey,
4876 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004877
4878 video_stream_encoder_->Stop();
4879}
4880
4881TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
4882 // Configure internal source factory and setup test again.
4883 encoder_factory_.SetHasInternalSource(true);
4884 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004885 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004886 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004887 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004888
4889 // Call encoder directly, simulating internal source where encoded frame
4890 // callback in VideoStreamEncoder is called despite no OnFrame().
4891 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
4892 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004893 EXPECT_THAT(
4894 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004895 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004896
Niels Möller8f7ce222019-03-21 15:43:58 +01004897 const std::vector<VideoFrameType> kDeltaFrame = {
4898 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01004899 // Need to set timestamp manually since manually for injected frame.
4900 VideoFrame frame = CreateFrame(101, nullptr);
4901 frame.set_timestamp(101);
4902 fake_encoder_.InjectFrame(frame, false);
4903 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004904 EXPECT_THAT(
4905 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004906 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004907
4908 // Request key-frame. The forces a dummy frame down into the encoder.
4909 fake_encoder_.ExpectNullFrame();
4910 video_stream_encoder_->SendKeyFrame();
4911 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01004912 EXPECT_THAT(
4913 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004914 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004915
4916 video_stream_encoder_->Stop();
4917}
Erik Språngb7cb7b52019-02-26 15:52:33 +01004918
4919TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
4920 // Configure internal source factory and setup test again.
4921 encoder_factory_.SetHasInternalSource(true);
4922 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02004923 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02004924 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01004925 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01004926
4927 int64_t timestamp = 1;
4928 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02004929 image.SetEncodedData(
4930 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01004931 image.capture_time_ms_ = ++timestamp;
4932 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
4933 const int64_t kEncodeFinishDelayMs = 10;
4934 image.timing_.encode_start_ms = timestamp;
4935 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
4936 fake_encoder_.InjectEncodedImage(image);
4937 // Wait for frame without incrementing clock.
4938 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4939 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
4940 // capture timestamp should be kEncodeFinishDelayMs in the past.
4941 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
4942 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
4943 kEncodeFinishDelayMs);
4944
4945 video_stream_encoder_->Stop();
4946}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02004947
4948TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
4949 // Configure internal source factory and setup test again.
4950 encoder_factory_.SetHasInternalSource(true);
4951 ResetEncoder("H264", 1, 1, 1, false);
4952
4953 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
4954 image._frameType = VideoFrameType::kVideoFrameKey;
4955
4956 CodecSpecificInfo codec_specific_info;
4957 codec_specific_info.codecType = kVideoCodecH264;
4958
4959 RTPFragmentationHeader fragmentation;
4960 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4961 fragmentation.fragmentationOffset[0] = 4;
4962 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
4963
4964 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4965 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4966
4967 EXPECT_THAT(sink_.GetLastEncodedImageData(),
4968 testing::ElementsAreArray(optimal_sps));
4969 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
4970 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
4971 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
4972 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
4973
4974 video_stream_encoder_->Stop();
4975}
4976
4977TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
4978 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
4979 0x00, 0x00, 0x03, 0x03, 0xF4,
4980 0x05, 0x03, 0xC7, 0xC0};
4981
4982 // Configure internal source factory and setup test again.
4983 encoder_factory_.SetHasInternalSource(true);
4984 ResetEncoder("H264", 1, 1, 1, false);
4985
4986 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
4987 image._frameType = VideoFrameType::kVideoFrameKey;
4988
4989 CodecSpecificInfo codec_specific_info;
4990 codec_specific_info.codecType = kVideoCodecH264;
4991
4992 RTPFragmentationHeader fragmentation;
4993 fragmentation.VerifyAndAllocateFragmentationHeader(1);
4994 fragmentation.fragmentationOffset[0] = 4;
4995 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
4996
4997 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
4998 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
4999
5000 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5001 testing::ElementsAreArray(optimal_sps));
5002 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5003 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5004 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5005 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5006
5007 video_stream_encoder_->Stop();
5008}
5009
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005010TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5011 const int kFrameWidth = 1280;
5012 const int kFrameHeight = 720;
5013 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5014
5015 video_stream_encoder_->OnBitrateUpdated(
Florent Castellia8336d32019-09-09 13:36:55 +02005016 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01005017 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005018 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5019
5020 // Insert a first video frame. It should be dropped because of downscale in
5021 // resolution.
5022 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5023 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5024 frame.set_rotation(kVideoRotation_270);
5025 video_source_.IncomingCapturedFrame(frame);
5026
5027 ExpectDroppedFrame();
5028
5029 // Second frame is downscaled.
5030 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5031 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5032 frame.set_rotation(kVideoRotation_90);
5033 video_source_.IncomingCapturedFrame(frame);
5034
5035 WaitForEncodedFrame(timestamp_ms);
5036 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5037
5038 // Insert another frame, also downscaled.
5039 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5040 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5041 frame.set_rotation(kVideoRotation_180);
5042 video_source_.IncomingCapturedFrame(frame);
5043
5044 WaitForEncodedFrame(timestamp_ms);
5045 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5046
5047 video_stream_encoder_->Stop();
5048}
5049
Erik Språng5056af02019-09-02 15:53:11 +02005050TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5051 const int kFrameWidth = 320;
5052 const int kFrameHeight = 180;
5053
5054 // Initial rate.
5055 video_stream_encoder_->OnBitrateUpdated(
5056 /*target_bitrate=*/DataRate::kbps(300),
Florent Castellia8336d32019-09-09 13:36:55 +02005057 /*stable_target_bitrate=*/DataRate::kbps(300),
Erik Språng5056af02019-09-02 15:53:11 +02005058 /*link_allocation=*/DataRate::kbps(300),
5059 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005060 /*rtt_ms=*/0,
5061 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005062
5063 // Insert a first video frame so that encoder gets configured.
5064 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5065 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5066 frame.set_rotation(kVideoRotation_270);
5067 video_source_.IncomingCapturedFrame(frame);
5068 WaitForEncodedFrame(timestamp_ms);
5069
5070 // Set a target rate below the minimum allowed by the codec settings.
5071 VideoCodec codec_config = fake_encoder_.codec_config();
5072 DataRate min_rate = DataRate::kbps(codec_config.minBitrate);
5073 DataRate target_rate = min_rate - DataRate::kbps(1);
5074 video_stream_encoder_->OnBitrateUpdated(
5075 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005076 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005077 /*link_allocation=*/target_rate,
5078 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005079 /*rtt_ms=*/0,
5080 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005081 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5082
5083 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5084 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5085 ASSERT_TRUE(rate_settings.has_value());
5086 DataRate allocation_sum = DataRate::bps(rate_settings->bitrate.get_sum_bps());
5087 EXPECT_EQ(min_rate, allocation_sum);
5088 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5089
5090 video_stream_encoder_->Stop();
5091}
5092
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005093TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005094 video_stream_encoder_->OnBitrateUpdated(
5095 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01005096 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005097 // Capture a frame and wait for it to synchronize with the encoder thread.
5098 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5099 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5100 WaitForEncodedFrame(1);
5101
5102 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5103 ASSERT_TRUE(prev_rate_settings.has_value());
5104 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5105 kDefaultFramerate);
5106
5107 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5108 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5109 timestamp_ms += 1000 / kDefaultFramerate;
5110 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5111 WaitForEncodedFrame(timestamp_ms);
5112 }
5113 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5114 kDefaultFramerate);
5115 // Capture larger frame to trigger a reconfigure.
5116 codec_height_ *= 2;
5117 codec_width_ *= 2;
5118 timestamp_ms += 1000 / kDefaultFramerate;
5119 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5120 WaitForEncodedFrame(timestamp_ms);
5121
5122 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5123 auto current_rate_settings =
5124 fake_encoder_.GetAndResetLastRateControlSettings();
5125 // Ensure we have actually reconfigured twice
5126 // The rate settings should have been set again even though
5127 // they haven't changed.
5128 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005129 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005130
5131 video_stream_encoder_->Stop();
5132}
5133
philipeld9cc8c02019-09-16 14:53:40 +02005134struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5135 MOCK_METHOD0(RequestEncoderFallback, void());
5136 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 11:30:00 +01005137 MOCK_METHOD1(RequestEncoderSwitch,
5138 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 14:53:40 +02005139};
5140
5141TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5142 constexpr int kDontCare = 100;
5143
5144 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5145 video_send_config_.encoder_settings.encoder_switch_request_callback =
5146 &switch_callback;
5147 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5148 encoder_config.codec_type = kVideoCodecVP8;
5149 webrtc::test::ScopedFieldTrials field_trial(
5150 "WebRTC-NetworkCondition-EncoderSwitch/"
5151 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5152 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5153
5154 // Reset encoder for new configuration to take effect.
5155 ConfigureEncoder(std::move(encoder_config));
5156
5157 // Send one frame to trigger ReconfigureEncoder.
5158 video_source_.IncomingCapturedFrame(
5159 CreateFrame(kDontCare, kDontCare, kDontCare));
5160
5161 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005162 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5163 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005164 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005165 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005166
5167 video_stream_encoder_->OnBitrateUpdated(
5168 /*target_bitrate=*/DataRate::kbps(50),
5169 /*stable_target_bitrate=*/DataRate::kbps(kDontCare),
5170 /*link_allocation=*/DataRate::kbps(kDontCare),
5171 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005172 /*rtt_ms=*/0,
5173 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005174
5175 video_stream_encoder_->Stop();
5176}
5177
5178TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5179 constexpr int kSufficientBitrateToNotDrop = 1000;
5180 constexpr int kHighRes = 500;
5181 constexpr int kLowRes = 100;
5182
5183 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5184 video_send_config_.encoder_settings.encoder_switch_request_callback =
5185 &switch_callback;
5186 webrtc::test::ScopedFieldTrials field_trial(
5187 "WebRTC-NetworkCondition-EncoderSwitch/"
5188 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5189 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5190 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5191 encoder_config.codec_type = kVideoCodecH264;
5192
5193 // Reset encoder for new configuration to take effect.
5194 ConfigureEncoder(std::move(encoder_config));
5195
5196 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5197 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5198 // not fail.
5199 video_stream_encoder_->OnBitrateUpdated(
5200 /*target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5201 /*stable_target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5202 /*link_allocation=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5203 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005204 /*rtt_ms=*/0,
5205 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005206
5207 // Send one frame to trigger ReconfigureEncoder.
5208 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5209 WaitForEncodedFrame(1);
5210
5211 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005212 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5213 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005214 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005215 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005216
5217 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5218 WaitForEncodedFrame(2);
5219
5220 video_stream_encoder_->Stop();
5221}
5222
philipel9b058032020-02-10 11:30:00 +01005223TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5224 constexpr int kDontCare = 100;
5225 StrictMock<MockEncoderSelector> encoder_selector;
5226 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5227 &fake_encoder_, &encoder_selector);
5228 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5229
5230 // Reset encoder for new configuration to take effect.
5231 ConfigureEncoder(video_encoder_config_.Copy());
5232
5233 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5234
5235 video_source_.IncomingCapturedFrame(
5236 CreateFrame(kDontCare, kDontCare, kDontCare));
5237 video_stream_encoder_->Stop();
5238
5239 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5240 // to it's factory, so in order for the encoder instance in the
5241 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5242 // reset the |video_stream_encoder_| here.
5243 video_stream_encoder_.reset();
5244}
5245
5246TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5247 constexpr int kDontCare = 100;
5248
5249 NiceMock<MockEncoderSelector> encoder_selector;
5250 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5251 video_send_config_.encoder_settings.encoder_switch_request_callback =
5252 &switch_callback;
5253 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5254 &fake_encoder_, &encoder_selector);
5255 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5256
5257 // Reset encoder for new configuration to take effect.
5258 ConfigureEncoder(video_encoder_config_.Copy());
5259
5260 ON_CALL(encoder_selector, OnEncodingBitrate(_))
5261 .WillByDefault(Return(SdpVideoFormat("AV1")));
5262 EXPECT_CALL(switch_callback,
5263 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5264 Field(&SdpVideoFormat::name, "AV1"))));
5265
5266 video_stream_encoder_->OnBitrateUpdated(
5267 /*target_bitrate=*/DataRate::kbps(50),
5268 /*stable_target_bitrate=*/DataRate::kbps(kDontCare),
5269 /*link_allocation=*/DataRate::kbps(kDontCare),
5270 /*fraction_lost=*/0,
5271 /*rtt_ms=*/0,
5272 /*cwnd_reduce_ratio=*/0);
5273
5274 video_stream_encoder_->Stop();
5275}
5276
5277TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5278 constexpr int kSufficientBitrateToNotDrop = 1000;
5279 constexpr int kDontCare = 100;
5280
5281 NiceMock<MockVideoEncoder> video_encoder;
5282 NiceMock<MockEncoderSelector> encoder_selector;
5283 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5284 video_send_config_.encoder_settings.encoder_switch_request_callback =
5285 &switch_callback;
5286 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5287 &video_encoder, &encoder_selector);
5288 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5289
5290 // Reset encoder for new configuration to take effect.
5291 ConfigureEncoder(video_encoder_config_.Copy());
5292
5293 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5294 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5295 // not fail.
5296 video_stream_encoder_->OnBitrateUpdated(
5297 /*target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5298 /*stable_target_bitrate=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5299 /*link_allocation=*/DataRate::kbps(kSufficientBitrateToNotDrop),
5300 /*fraction_lost=*/0,
5301 /*rtt_ms=*/0,
5302 /*cwnd_reduce_ratio=*/0);
5303
5304 ON_CALL(video_encoder, Encode(_, _))
5305 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5306 ON_CALL(encoder_selector, OnEncoderBroken())
5307 .WillByDefault(Return(SdpVideoFormat("AV2")));
5308
5309 rtc::Event encode_attempted;
5310 EXPECT_CALL(switch_callback,
5311 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5312 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5313 EXPECT_EQ(format.name, "AV2");
5314 encode_attempted.Set();
5315 });
5316
5317 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5318 encode_attempted.Wait(3000);
5319
5320 video_stream_encoder_->Stop();
5321
5322 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5323 // to it's factory, so in order for the encoder instance in the
5324 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5325 // reset the |video_stream_encoder_| here.
5326 video_stream_encoder_.reset();
5327}
5328
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005329TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005330 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005331 const int kFrameWidth = 320;
5332 const int kFrameHeight = 180;
5333
5334 // Set initial rate.
5335 auto rate = DataRate::kbps(100);
5336 video_stream_encoder_->OnBitrateUpdated(
5337 /*target_bitrate=*/rate,
5338 /*stable_target_bitrate=*/rate,
5339 /*link_allocation=*/rate,
5340 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005341 /*rtt_ms=*/0,
5342 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005343
5344 // Insert a first video frame so that encoder gets configured.
5345 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5346 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5347 frame.set_rotation(kVideoRotation_270);
5348 video_source_.IncomingCapturedFrame(frame);
5349 WaitForEncodedFrame(timestamp_ms);
5350 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5351
5352 // Change of target bitrate propagates to the encoder.
5353 auto new_stable_rate = rate - DataRate::kbps(5);
5354 video_stream_encoder_->OnBitrateUpdated(
5355 /*target_bitrate=*/new_stable_rate,
5356 /*stable_target_bitrate=*/new_stable_rate,
5357 /*link_allocation=*/rate,
5358 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005359 /*rtt_ms=*/0,
5360 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005361 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5362 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5363 video_stream_encoder_->Stop();
5364}
5365
5366TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005367 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005368 const int kFrameWidth = 320;
5369 const int kFrameHeight = 180;
5370
5371 // Set initial rate.
5372 auto rate = DataRate::kbps(100);
5373 video_stream_encoder_->OnBitrateUpdated(
5374 /*target_bitrate=*/rate,
5375 /*stable_target_bitrate=*/rate,
5376 /*link_allocation=*/rate,
5377 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005378 /*rtt_ms=*/0,
5379 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005380
5381 // Insert a first video frame so that encoder gets configured.
5382 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5383 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5384 frame.set_rotation(kVideoRotation_270);
5385 video_source_.IncomingCapturedFrame(frame);
5386 WaitForEncodedFrame(timestamp_ms);
5387 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5388
5389 // Set a higher target rate without changing the link_allocation. Should not
5390 // reset encoder's rate.
5391 auto new_stable_rate = rate - DataRate::kbps(5);
5392 video_stream_encoder_->OnBitrateUpdated(
5393 /*target_bitrate=*/rate,
5394 /*stable_target_bitrate=*/new_stable_rate,
5395 /*link_allocation=*/rate,
5396 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005397 /*rtt_ms=*/0,
5398 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005399 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5400 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5401 video_stream_encoder_->Stop();
5402}
5403
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005404TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5405 test::ScopedFieldTrials field_trials(
5406 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5407 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5408 const int kFramerateFps = 30;
5409 const int kWidth = 1920;
5410 const int kHeight = 1080;
5411 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5412 // Works on screenshare mode.
5413 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5414 // We rely on the automatic resolution adaptation, but we handle framerate
5415 // adaptation manually by mocking the stats proxy.
5416 video_source_.set_adaptation_enabled(true);
5417
5418 // BALANCED degradation preference is required for this feature.
5419 video_stream_encoder_->OnBitrateUpdated(
5420 DataRate::bps(kTargetBitrateBps), DataRate::bps(kTargetBitrateBps),
Ying Wang9b881ab2020-02-07 14:29:32 +01005421 DataRate::bps(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005422 video_stream_encoder_->SetSource(&video_source_,
5423 webrtc::DegradationPreference::BALANCED);
5424 VerifyNoLimitation(video_source_.sink_wants());
5425
5426 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5427 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5428
5429 // Pass enough frames with the full update to trigger animation detection.
5430 for (int i = 0; i < kNumFrames; ++i) {
5431 int64_t timestamp_ms =
5432 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5433 frame.set_ntp_time_ms(timestamp_ms);
5434 frame.set_timestamp_us(timestamp_ms * 1000);
5435 video_source_.IncomingCapturedFrame(frame);
5436 WaitForEncodedFrame(timestamp_ms);
5437 }
5438
5439 // Resolution should be limited.
5440 rtc::VideoSinkWants expected;
5441 expected.max_framerate_fps = kFramerateFps;
5442 expected.max_pixel_count = 1280 * 720 + 1;
5443 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5444
5445 // Pass one frame with no known update.
5446 // Resolution cap should be removed immediately.
5447 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5448 frame.set_ntp_time_ms(timestamp_ms);
5449 frame.set_timestamp_us(timestamp_ms * 1000);
5450 frame.clear_update_rect();
5451
5452 video_source_.IncomingCapturedFrame(frame);
5453 WaitForEncodedFrame(timestamp_ms);
5454
5455 // Resolution should be unlimited now.
5456 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5457
5458 video_stream_encoder_->Stop();
5459}
5460
perkj26091b12016-09-01 01:17:40 -07005461} // namespace webrtc