blob: 38bd28b24b7eff2b695b90b982976bff2d10c988 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010034#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080035#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020036#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010038#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020039#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "system_wrappers/include/sleep.h"
41#include "test/encoder_settings.h"
42#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020043#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010044#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020045#include "test/gmock.h"
46#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020047#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020048#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070049
50namespace webrtc {
51
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020052using ScaleReason = VideoAdaptationReason;
sprang57c2fff2017-01-16 06:24:02 -080053using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020054using ::testing::AllOf;
55using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010056using ::testing::Matcher;
57using ::testing::NiceMock;
58using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080060
perkj803d97f2016-11-01 11:45:46 -070061namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020062const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010063const int kQpLow = 1;
64const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020065const int kMinFramerateFps = 2;
66const int kMinBalancedFramerateFps = 7;
67const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080068const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010069const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020070const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010071const uint32_t kSimulcastTargetBitrateBps = 3150000;
72const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080073const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070074const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020076const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020077const VideoEncoder::ResolutionBitrateLimits
78 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
79const VideoEncoder::ResolutionBitrateLimits
80 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080081
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020082uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
83 0x00, 0x00, 0x03, 0x03, 0xF4,
84 0x05, 0x03, 0xC7, 0xE0, 0x1B,
85 0x41, 0x10, 0x8D, 0x00};
86
perkj803d97f2016-11-01 11:45:46 -070087class TestBuffer : public webrtc::I420Buffer {
88 public:
89 TestBuffer(rtc::Event* event, int width, int height)
90 : I420Buffer(width, height), event_(event) {}
91
92 private:
93 friend class rtc::RefCountedObject<TestBuffer>;
94 ~TestBuffer() override {
95 if (event_)
96 event_->Set();
97 }
98 rtc::Event* const event_;
99};
100
Noah Richards51db4212019-06-12 06:59:12 -0700101// A fake native buffer that can't be converted to I420.
102class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
103 public:
104 FakeNativeBuffer(rtc::Event* event, int width, int height)
105 : event_(event), width_(width), height_(height) {}
106 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
107 int width() const override { return width_; }
108 int height() const override { return height_; }
109 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
110 return nullptr;
111 }
112
113 private:
114 friend class rtc::RefCountedObject<FakeNativeBuffer>;
115 ~FakeNativeBuffer() override {
116 if (event_)
117 event_->Set();
118 }
119 rtc::Event* const event_;
120 const int width_;
121 const int height_;
122};
123
Niels Möller7dc26b72017-12-06 10:27:48 +0100124class CpuOveruseDetectorProxy : public OveruseFrameDetector {
125 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200126 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
127 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100128 last_target_framerate_fps_(-1) {}
129 virtual ~CpuOveruseDetectorProxy() {}
130
131 void OnTargetFramerateUpdated(int framerate_fps) override {
132 rtc::CritScope cs(&lock_);
133 last_target_framerate_fps_ = framerate_fps;
134 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
135 }
136
137 int GetLastTargetFramerate() {
138 rtc::CritScope cs(&lock_);
139 return last_target_framerate_fps_;
140 }
141
Niels Möller4db138e2018-04-19 09:04:13 +0200142 CpuOveruseOptions GetOptions() { return options_; }
143
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 private:
145 rtc::CriticalSection lock_;
146 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
147};
148
mflodmancc3d4422017-08-03 08:27:51 -0700149class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700150 public:
Niels Möller213618e2018-07-24 09:29:58 +0200151 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200152 const VideoStreamEncoderSettings& settings,
153 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100154 : VideoStreamEncoder(Clock::GetRealTimeClock(),
155 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200156 stats_proxy,
157 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200158 std::unique_ptr<OveruseFrameDetector>(
159 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100160 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100161 task_queue_factory),
Henrik Boströmdc4f75f2020-04-20 12:04:12 +0200162 fake_cpu_resource_(std::make_unique<FakeResource>("FakeResource[CPU]")),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100163 fake_quality_resource_(
Henrik Boströmdc4f75f2020-04-20 12:04:12 +0200164 std::make_unique<FakeResource>("FakeResource[QP]")) {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200165 InjectAdaptationResource(fake_quality_resource_.get(),
166 VideoAdaptationReason::kQuality);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100167 InjectAdaptationResource(fake_cpu_resource_.get(),
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200168 VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100169 }
perkj803d97f2016-11-01 11:45:46 -0700170
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200171 void PostTaskAndWait(bool down, VideoAdaptationReason reason) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200172 PostTaskAndWait(down, reason, /*expected_results=*/true);
173 }
174
Henrik Boströmb08882b2020-01-07 10:11:17 +0100175 void PostTaskAndWait(bool down,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200176 VideoAdaptationReason reason,
Henrik Boströmb08882b2020-01-07 10:11:17 +0100177 bool expected_results) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100178 rtc::Event event;
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200179 encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100180 ResourceUsageState usage_state =
181 down ? ResourceUsageState::kOveruse : ResourceUsageState::kUnderuse;
182
183 FakeResource* resource = nullptr;
184 switch (reason) {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200185 case VideoAdaptationReason::kQuality:
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100186 resource = fake_quality_resource_.get();
187 break;
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200188 case VideoAdaptationReason::kCpu:
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100189 resource = fake_cpu_resource_.get();
190 break;
191 default:
192 RTC_NOTREACHED();
193 }
194
195 resource->set_usage_state(usage_state);
196 if (!expected_results) {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200197 ASSERT_EQ(VideoAdaptationReason::kQuality, reason)
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100198 << "We can only assert adaptation result for quality resources";
199 EXPECT_EQ(
200 ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency,
201 resource->last_response());
202 } else {
203 EXPECT_EQ(ResourceListenerResponse::kNothing,
204 resource->last_response());
205 }
206
perkj803d97f2016-11-01 11:45:46 -0700207 event.Set();
208 });
perkj070ba852017-02-16 15:46:27 -0800209 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700210 }
211
kthelgason2fc52542017-03-03 00:24:41 -0800212 // This is used as a synchronisation mechanism, to make sure that the
213 // encoder queue is not blocked before we start sending it frames.
214 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100215 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200216 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800217 ASSERT_TRUE(event.Wait(5000));
218 }
219
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200220 void TriggerCpuOveruse() {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200221 PostTaskAndWait(/*down=*/true, VideoAdaptationReason::kCpu);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200222 }
kthelgason876222f2016-11-29 01:44:11 -0800223
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200224 void TriggerCpuNormalUsage() {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200225 PostTaskAndWait(/*down=*/false, VideoAdaptationReason::kCpu);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200226 }
kthelgason876222f2016-11-29 01:44:11 -0800227
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200228 void TriggerQualityLow() {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200229 PostTaskAndWait(/*down=*/true, VideoAdaptationReason::kQuality);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200230 }
kthelgason876222f2016-11-29 01:44:11 -0800231
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200232 void TriggerQualityLowExpectFalse() {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200233 PostTaskAndWait(/*down=*/true, VideoAdaptationReason::kQuality,
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200234 /*expected_results=*/false);
235 }
236
237 void TriggerQualityHigh() {
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200238 PostTaskAndWait(/*down=*/false, VideoAdaptationReason::kQuality);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200239 }
sprangfda496a2017-06-15 04:21:07 -0700240
Niels Möller7dc26b72017-12-06 10:27:48 +0100241 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100242 std::unique_ptr<FakeResource> fake_cpu_resource_;
243 std::unique_ptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700244};
245
asapersson5f7226f2016-11-25 04:37:00 -0800246class VideoStreamFactory
247 : public VideoEncoderConfig::VideoStreamFactoryInterface {
248 public:
sprangfda496a2017-06-15 04:21:07 -0700249 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
250 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800251 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700252 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800253 }
254
255 private:
256 std::vector<VideoStream> CreateEncoderStreams(
257 int width,
258 int height,
259 const VideoEncoderConfig& encoder_config) override {
260 std::vector<VideoStream> streams =
261 test::CreateVideoStreams(width, height, encoder_config);
262 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100263 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700264 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800265 }
266 return streams;
267 }
sprangfda496a2017-06-15 04:21:07 -0700268
asapersson5f7226f2016-11-25 04:37:00 -0800269 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700270 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800271};
272
Noah Richards51db4212019-06-12 06:59:12 -0700273// Simulates simulcast behavior and makes highest stream resolutions divisible
274// by 4.
275class CroppingVideoStreamFactory
276 : public VideoEncoderConfig::VideoStreamFactoryInterface {
277 public:
278 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
279 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
280 EXPECT_GT(num_temporal_layers, 0u);
281 EXPECT_GT(framerate, 0);
282 }
283
284 private:
285 std::vector<VideoStream> CreateEncoderStreams(
286 int width,
287 int height,
288 const VideoEncoderConfig& encoder_config) override {
289 std::vector<VideoStream> streams = test::CreateVideoStreams(
290 width - width % 4, height - height % 4, encoder_config);
291 for (VideoStream& stream : streams) {
292 stream.num_temporal_layers = num_temporal_layers_;
293 stream.max_framerate = framerate_;
294 }
295 return streams;
296 }
297
298 const size_t num_temporal_layers_;
299 const int framerate_;
300};
301
sprangb1ca0732017-02-01 08:38:12 -0800302class AdaptingFrameForwarder : public test::FrameForwarder {
303 public:
304 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700305 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800306
307 void set_adaptation_enabled(bool enabled) {
308 rtc::CritScope cs(&crit_);
309 adaptation_enabled_ = enabled;
310 }
311
asaperssonfab67072017-04-04 05:51:49 -0700312 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800313 rtc::CritScope cs(&crit_);
314 return adaptation_enabled_;
315 }
316
asapersson09f05612017-05-15 23:40:18 -0700317 rtc::VideoSinkWants last_wants() const {
318 rtc::CritScope cs(&crit_);
319 return last_wants_;
320 }
321
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200322 absl::optional<int> last_sent_width() const { return last_width_; }
323 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800324
sprangb1ca0732017-02-01 08:38:12 -0800325 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
326 int cropped_width = 0;
327 int cropped_height = 0;
328 int out_width = 0;
329 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700330 if (adaption_enabled()) {
331 if (adapter_.AdaptFrameResolution(
332 video_frame.width(), video_frame.height(),
333 video_frame.timestamp_us() * 1000, &cropped_width,
334 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100335 VideoFrame adapted_frame =
336 VideoFrame::Builder()
337 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
338 nullptr, out_width, out_height))
339 .set_timestamp_rtp(99)
340 .set_timestamp_ms(99)
341 .set_rotation(kVideoRotation_0)
342 .build();
sprangc5d62e22017-04-02 23:53:04 -0700343 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100344 if (video_frame.has_update_rect()) {
345 adapted_frame.set_update_rect(
346 video_frame.update_rect().ScaleWithFrame(
347 video_frame.width(), video_frame.height(), 0, 0,
348 video_frame.width(), video_frame.height(), out_width,
349 out_height));
350 }
sprangc5d62e22017-04-02 23:53:04 -0700351 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800352 last_width_.emplace(adapted_frame.width());
353 last_height_.emplace(adapted_frame.height());
354 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200355 last_width_ = absl::nullopt;
356 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700357 }
sprangb1ca0732017-02-01 08:38:12 -0800358 } else {
359 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800360 last_width_.emplace(video_frame.width());
361 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800362 }
363 }
364
365 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
366 const rtc::VideoSinkWants& wants) override {
367 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700368 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100369 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800370 test::FrameForwarder::AddOrUpdateSink(sink, wants);
371 }
sprangb1ca0732017-02-01 08:38:12 -0800372 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700373 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
374 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200375 absl::optional<int> last_width_;
376 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800377};
sprangc5d62e22017-04-02 23:53:04 -0700378
Niels Möller213618e2018-07-24 09:29:58 +0200379// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700380class MockableSendStatisticsProxy : public SendStatisticsProxy {
381 public:
382 MockableSendStatisticsProxy(Clock* clock,
383 const VideoSendStream::Config& config,
384 VideoEncoderConfig::ContentType content_type)
385 : SendStatisticsProxy(clock, config, content_type) {}
386
387 VideoSendStream::Stats GetStats() override {
388 rtc::CritScope cs(&lock_);
389 if (mock_stats_)
390 return *mock_stats_;
391 return SendStatisticsProxy::GetStats();
392 }
393
Niels Möller213618e2018-07-24 09:29:58 +0200394 int GetInputFrameRate() const override {
395 rtc::CritScope cs(&lock_);
396 if (mock_stats_)
397 return mock_stats_->input_frame_rate;
398 return SendStatisticsProxy::GetInputFrameRate();
399 }
sprangc5d62e22017-04-02 23:53:04 -0700400 void SetMockStats(const VideoSendStream::Stats& stats) {
401 rtc::CritScope cs(&lock_);
402 mock_stats_.emplace(stats);
403 }
404
405 void ResetMockStats() {
406 rtc::CritScope cs(&lock_);
407 mock_stats_.reset();
408 }
409
410 private:
411 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200412 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700413};
414
sprang4847ae62017-06-27 07:06:52 -0700415class MockBitrateObserver : public VideoBitrateAllocationObserver {
416 public:
Erik Språng566124a2018-04-23 12:32:22 +0200417 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700418};
419
philipel9b058032020-02-10 11:30:00 +0100420class MockEncoderSelector
421 : public VideoEncoderFactory::EncoderSelectorInterface {
422 public:
423 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
Mirta Dvornicic4f34d782020-02-26 13:01:19 +0100424 MOCK_METHOD1(OnAvailableBitrate,
philipel9b058032020-02-10 11:30:00 +0100425 absl::optional<SdpVideoFormat>(const DataRate& rate));
426 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
427};
428
perkj803d97f2016-11-01 11:45:46 -0700429} // namespace
430
mflodmancc3d4422017-08-03 08:27:51 -0700431class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700432 public:
433 static const int kDefaultTimeoutMs = 30 * 1000;
434
mflodmancc3d4422017-08-03 08:27:51 -0700435 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700436 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700437 codec_width_(320),
438 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200439 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200440 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700441 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200442 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700443 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700444 Clock::GetRealTimeClock(),
445 video_send_config_,
446 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700447 sink_(&fake_encoder_) {}
448
449 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700450 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700451 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200452 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800453 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200454 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200455 video_send_config_.rtp.payload_name = "FAKE";
456 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700457
Per512ecb32016-09-23 15:52:06 +0200458 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200459 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700460 video_encoder_config.video_stream_factory =
461 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100462 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700463
464 // Framerate limit is specified by the VideoStreamFactory.
465 std::vector<VideoStream> streams =
466 video_encoder_config.video_stream_factory->CreateEncoderStreams(
467 codec_width_, codec_height_, video_encoder_config);
468 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100469 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700470
Niels Möllerf1338562018-04-26 09:51:47 +0200471 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800472 }
473
Niels Möllerf1338562018-04-26 09:51:47 +0200474 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700475 if (video_stream_encoder_)
476 video_stream_encoder_->Stop();
477 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200478 stats_proxy_.get(), video_send_config_.encoder_settings,
479 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700480 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
481 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700482 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700483 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
484 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200485 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700486 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800487 }
488
489 void ResetEncoder(const std::string& payload_name,
490 size_t num_streams,
491 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700492 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700493 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200494 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800495
496 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200497 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800498 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100499 video_encoder_config.max_bitrate_bps =
500 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800501 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700502 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
503 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700504 video_encoder_config.content_type =
505 screenshare ? VideoEncoderConfig::ContentType::kScreen
506 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700507 if (payload_name == "VP9") {
508 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
509 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
510 video_encoder_config.encoder_specific_settings =
511 new rtc::RefCountedObject<
512 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
513 }
Niels Möllerf1338562018-04-26 09:51:47 +0200514 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700515 }
516
sprang57c2fff2017-01-16 06:24:02 -0800517 VideoFrame CreateFrame(int64_t ntp_time_ms,
518 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100519 VideoFrame frame =
520 VideoFrame::Builder()
521 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
522 destruction_event, codec_width_, codec_height_))
523 .set_timestamp_rtp(99)
524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
sprang57c2fff2017-01-16 06:24:02 -0800527 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700528 return frame;
529 }
530
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100531 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
532 rtc::Event* destruction_event,
533 int offset_x) const {
534 VideoFrame frame =
535 VideoFrame::Builder()
536 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
537 destruction_event, codec_width_, codec_height_))
538 .set_timestamp_rtp(99)
539 .set_timestamp_ms(99)
540 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100541 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100542 .build();
543 frame.set_ntp_time_ms(ntp_time_ms);
544 return frame;
545 }
546
sprang57c2fff2017-01-16 06:24:02 -0800547 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100548 VideoFrame frame =
549 VideoFrame::Builder()
550 .set_video_frame_buffer(
551 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
552 .set_timestamp_rtp(99)
553 .set_timestamp_ms(99)
554 .set_rotation(kVideoRotation_0)
555 .build();
sprang57c2fff2017-01-16 06:24:02 -0800556 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700557 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700558 return frame;
559 }
560
Noah Richards51db4212019-06-12 06:59:12 -0700561 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
562 rtc::Event* destruction_event,
563 int width,
564 int height) const {
565 VideoFrame frame =
566 VideoFrame::Builder()
567 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
568 destruction_event, width, height))
569 .set_timestamp_rtp(99)
570 .set_timestamp_ms(99)
571 .set_rotation(kVideoRotation_0)
572 .build();
573 frame.set_ntp_time_ms(ntp_time_ms);
574 return frame;
575 }
576
577 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
578 rtc::Event* destruction_event) const {
579 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
580 codec_height_);
581 }
582
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100583 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
584 MockBitrateObserver bitrate_observer;
585 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
586
587 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
588 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200589 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100590 DataRate::BitsPerSec(kTargetBitrateBps),
591 DataRate::BitsPerSec(kTargetBitrateBps),
592 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100593
594 video_source_.IncomingCapturedFrame(
595 CreateFrame(1, codec_width_, codec_height_));
596 WaitForEncodedFrame(1);
597 }
598
asapersson02465b82017-04-10 01:12:52 -0700599 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700600 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700601 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
602 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700603 }
604
asapersson09f05612017-05-15 23:40:18 -0700605 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
606 const rtc::VideoSinkWants& wants2) {
607 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
608 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
609 }
610
Åsa Persson8c1bf952018-09-13 10:42:19 +0200611 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
612 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
613 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
614 EXPECT_FALSE(wants.target_pixel_count);
615 }
616
asapersson09f05612017-05-15 23:40:18 -0700617 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
618 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200619 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700620 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
621 EXPECT_GT(wants1.max_pixel_count, 0);
622 }
623
624 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
625 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200626 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700627 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
628 }
629
asaperssonf7e294d2017-06-13 23:25:22 -0700630 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
631 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200632 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700633 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
634 }
635
636 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
637 const rtc::VideoSinkWants& wants2) {
638 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
639 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
640 }
641
642 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
643 const rtc::VideoSinkWants& wants2) {
644 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
645 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
646 }
647
648 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
649 const rtc::VideoSinkWants& wants2) {
650 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
651 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
652 EXPECT_GT(wants1.max_pixel_count, 0);
653 }
654
655 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
656 const rtc::VideoSinkWants& wants2) {
657 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
658 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
659 }
660
asapersson09f05612017-05-15 23:40:18 -0700661 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
662 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200663 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700664 EXPECT_LT(wants.max_pixel_count, pixel_count);
665 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700666 }
667
668 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
669 EXPECT_LT(wants.max_framerate_fps, fps);
670 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
671 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700672 }
673
asaperssonf7e294d2017-06-13 23:25:22 -0700674 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
675 int expected_fps) {
676 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
677 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
678 EXPECT_FALSE(wants.target_pixel_count);
679 }
680
Jonathan Yubc771b72017-12-08 17:04:29 -0800681 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
682 int last_frame_pixels) {
683 // Balanced mode should always scale FPS to the desired range before
684 // attempting to scale resolution.
685 int fps_limit = wants.max_framerate_fps;
686 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100687 EXPECT_LE(7, fps_limit);
688 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800689 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100690 EXPECT_LE(10, fps_limit);
691 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800692 } else if (last_frame_pixels <= 640 * 480) {
693 EXPECT_LE(15, fps_limit);
694 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200695 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800696 }
697 }
698
sprang4847ae62017-06-27 07:06:52 -0700699 void WaitForEncodedFrame(int64_t expected_ntp_time) {
700 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100701 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700702 }
703
704 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
705 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100706 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700707 return ok;
708 }
709
710 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
711 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100712 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700713 }
714
715 void ExpectDroppedFrame() {
716 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100717 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700718 }
719
720 bool WaitForFrame(int64_t timeout_ms) {
721 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100722 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700723 return ok;
724 }
725
perkj26091b12016-09-01 01:17:40 -0700726 class TestEncoder : public test::FakeEncoder {
727 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100728 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700729
asaperssonfab67072017-04-04 05:51:49 -0700730 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800731 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700732 return config_;
733 }
734
735 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800736 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700737 block_next_encode_ = true;
738 }
739
Erik Språngaed30702018-11-05 12:57:17 +0100740 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800741 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100742 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100743 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100744 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100745 info.scaling_settings = VideoEncoder::ScalingSettings(
746 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100747 }
748 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100749 for (int i = 0; i < kMaxSpatialLayers; ++i) {
750 if (temporal_layers_supported_[i]) {
751 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
752 info.fps_allocation[i].resize(num_layers);
753 }
754 }
Erik Språngaed30702018-11-05 12:57:17 +0100755 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200756
757 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100758 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100759 return info;
kthelgason876222f2016-11-29 01:44:11 -0800760 }
761
Erik Språngb7cb7b52019-02-26 15:52:33 +0100762 int32_t RegisterEncodeCompleteCallback(
763 EncodedImageCallback* callback) override {
764 rtc::CritScope lock(&local_crit_sect_);
765 encoded_image_callback_ = callback;
766 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
767 }
768
perkjfa10b552016-10-02 23:45:26 -0700769 void ContinueEncode() { continue_encode_event_.Set(); }
770
771 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
772 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800773 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700774 EXPECT_EQ(timestamp_, timestamp);
775 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
776 }
777
kthelgason2fc52542017-03-03 00:24:41 -0800778 void SetQualityScaling(bool b) {
779 rtc::CritScope lock(&local_crit_sect_);
780 quality_scaling_ = b;
781 }
kthelgasonad9010c2017-02-14 00:46:51 -0800782
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100783 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
784 rtc::CritScope lock(&local_crit_sect_);
785 requested_resolution_alignment_ = requested_resolution_alignment;
786 }
787
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100788 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
789 rtc::CritScope lock(&local_crit_sect_);
790 is_hardware_accelerated_ = is_hardware_accelerated;
791 }
792
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100793 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
794 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
795 rtc::CritScope lock(&local_crit_sect_);
796 temporal_layers_supported_[spatial_idx] = supported;
797 }
798
Sergey Silkin6456e352019-07-08 17:56:40 +0200799 void SetResolutionBitrateLimits(
800 std::vector<ResolutionBitrateLimits> thresholds) {
801 rtc::CritScope cs(&local_crit_sect_);
802 resolution_bitrate_limits_ = thresholds;
803 }
804
sprangfe627f32017-03-29 08:24:59 -0700805 void ForceInitEncodeFailure(bool force_failure) {
806 rtc::CritScope lock(&local_crit_sect_);
807 force_init_encode_failed_ = force_failure;
808 }
809
Niels Möller6bb5ab92019-01-11 11:11:10 +0100810 void SimulateOvershoot(double rate_factor) {
811 rtc::CritScope lock(&local_crit_sect_);
812 rate_factor_ = rate_factor;
813 }
814
Erik Språngd7329ca2019-02-21 21:19:53 +0100815 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100816 rtc::CritScope lock(&local_crit_sect_);
817 return last_framerate_;
818 }
819
Erik Språngd7329ca2019-02-21 21:19:53 +0100820 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100821 rtc::CritScope lock(&local_crit_sect_);
822 return last_update_rect_;
823 }
824
Niels Möller87e2d782019-03-07 10:18:23 +0100825 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100826 rtc::CritScope lock(&local_crit_sect_);
827 return last_frame_types_;
828 }
829
830 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100831 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100832 keyframe ? VideoFrameType::kVideoFrameKey
833 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100834 {
835 rtc::CritScope lock(&local_crit_sect_);
836 last_frame_types_ = frame_type;
837 }
Niels Möllerb859b322019-03-07 12:40:01 +0100838 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100839 }
840
Erik Språngb7cb7b52019-02-26 15:52:33 +0100841 void InjectEncodedImage(const EncodedImage& image) {
842 rtc::CritScope lock(&local_crit_sect_);
843 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
844 }
845
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200846 void InjectEncodedImage(const EncodedImage& image,
847 const CodecSpecificInfo* codec_specific_info,
848 const RTPFragmentationHeader* fragmentation) {
849 rtc::CritScope lock(&local_crit_sect_);
850 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
851 fragmentation);
852 }
853
Erik Språngd7329ca2019-02-21 21:19:53 +0100854 void ExpectNullFrame() {
855 rtc::CritScope lock(&local_crit_sect_);
856 expect_null_frame_ = true;
857 }
858
Erik Språng5056af02019-09-02 15:53:11 +0200859 absl::optional<VideoEncoder::RateControlParameters>
860 GetAndResetLastRateControlSettings() {
861 auto settings = last_rate_control_settings_;
862 last_rate_control_settings_.reset();
863 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100864 }
865
Sergey Silkin5ee69672019-07-02 14:18:34 +0200866 int GetNumEncoderInitializations() const {
867 rtc::CritScope lock(&local_crit_sect_);
868 return num_encoder_initializations_;
869 }
870
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200871 int GetNumSetRates() const {
872 rtc::CritScope lock(&local_crit_sect_);
873 return num_set_rates_;
874 }
875
perkjfa10b552016-10-02 23:45:26 -0700876 private:
perkj26091b12016-09-01 01:17:40 -0700877 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100878 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700879 bool block_encode;
880 {
brandtre78d2662017-01-16 05:57:16 -0800881 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100882 if (expect_null_frame_) {
883 EXPECT_EQ(input_image.timestamp(), 0u);
884 EXPECT_EQ(input_image.width(), 1);
885 last_frame_types_ = *frame_types;
886 expect_null_frame_ = false;
887 } else {
888 EXPECT_GT(input_image.timestamp(), timestamp_);
889 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
890 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
891 }
perkj26091b12016-09-01 01:17:40 -0700892
893 timestamp_ = input_image.timestamp();
894 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700895 last_input_width_ = input_image.width();
896 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700897 block_encode = block_next_encode_;
898 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100899 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100900 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700901 }
Niels Möllerb859b322019-03-07 12:40:01 +0100902 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700903 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700904 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700905 return result;
906 }
907
sprangfe627f32017-03-29 08:24:59 -0700908 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200909 const Settings& settings) override {
910 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200911
sprangfe627f32017-03-29 08:24:59 -0700912 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100913 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200914
915 ++num_encoder_initializations_;
916
Erik Språng82fad3d2018-03-21 09:57:23 +0100917 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700918 // Simulate setting up temporal layers, in order to validate the life
919 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100920 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200921 frame_buffer_controller_ =
922 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700923 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100924 if (force_init_encode_failed_) {
925 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700926 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100927 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100928
Erik Språngb7cb7b52019-02-26 15:52:33 +0100929 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700930 return res;
931 }
932
Erik Språngb7cb7b52019-02-26 15:52:33 +0100933 int32_t Release() override {
934 rtc::CritScope lock(&local_crit_sect_);
935 EXPECT_NE(initialized_, EncoderState::kUninitialized);
936 initialized_ = EncoderState::kUninitialized;
937 return FakeEncoder::Release();
938 }
939
Erik Språng16cb8f52019-04-12 13:59:09 +0200940 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100941 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200942 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100943 VideoBitrateAllocation adjusted_rate_allocation;
944 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
945 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200946 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100947 adjusted_rate_allocation.SetBitrate(
948 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200949 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100950 rate_factor_));
951 }
952 }
953 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200954 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200955 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200956 RateControlParameters adjusted_paramters = parameters;
957 adjusted_paramters.bitrate = adjusted_rate_allocation;
958 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100959 }
960
brandtre78d2662017-01-16 05:57:16 -0800961 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100962 enum class EncoderState {
963 kUninitialized,
964 kInitializationFailed,
965 kInitialized
966 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
967 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700968 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700969 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700970 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
971 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
972 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
973 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
974 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100975 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100976 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100977 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700978 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100979 absl::optional<bool>
980 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
981 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700982 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100983 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
984 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200985 absl::optional<VideoEncoder::RateControlParameters>
986 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100987 VideoFrame::UpdateRect last_update_rect_
988 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100989 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100990 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100991 EncodedImageCallback* encoded_image_callback_
992 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +0100993 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200994 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +0200995 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
996 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200997 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -0700998 };
999
mflodmancc3d4422017-08-03 08:27:51 -07001000 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001001 public:
1002 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001003 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001004
perkj26091b12016-09-01 01:17:40 -07001005 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001006 EXPECT_TRUE(
1007 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1008 }
1009
1010 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1011 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001012 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001013 if (!encoded_frame_event_.Wait(timeout_ms))
1014 return false;
perkj26091b12016-09-01 01:17:40 -07001015 {
1016 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001017 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001018 }
1019 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001020 return true;
perkj26091b12016-09-01 01:17:40 -07001021 }
1022
sprangb1ca0732017-02-01 08:38:12 -08001023 void WaitForEncodedFrame(uint32_t expected_width,
1024 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001025 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001026 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001027 }
1028
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001029 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001030 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001031 uint32_t width = 0;
1032 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001033 {
1034 rtc::CritScope lock(&crit_);
1035 width = last_width_;
1036 height = last_height_;
1037 }
1038 EXPECT_EQ(expected_height, height);
1039 EXPECT_EQ(expected_width, width);
1040 }
1041
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001042 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1043 int width = 0;
1044 int height = 0;
1045 {
1046 rtc::CritScope lock(&crit_);
1047 width = last_width_;
1048 height = last_height_;
1049 }
1050 EXPECT_EQ(width % resolution_alignment, 0);
1051 EXPECT_EQ(height % resolution_alignment, 0);
1052 }
1053
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001054 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1055 VideoRotation rotation;
1056 {
1057 rtc::CritScope lock(&crit_);
1058 rotation = last_rotation_;
1059 }
1060 EXPECT_EQ(expected_rotation, rotation);
1061 }
1062
kthelgason2fc52542017-03-03 00:24:41 -08001063 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001064
sprangc5d62e22017-04-02 23:53:04 -07001065 bool WaitForFrame(int64_t timeout_ms) {
1066 return encoded_frame_event_.Wait(timeout_ms);
1067 }
1068
perkj26091b12016-09-01 01:17:40 -07001069 void SetExpectNoFrames() {
1070 rtc::CritScope lock(&crit_);
1071 expect_frames_ = false;
1072 }
1073
asaperssonfab67072017-04-04 05:51:49 -07001074 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001075 rtc::CritScope lock(&crit_);
1076 return number_of_reconfigurations_;
1077 }
1078
asaperssonfab67072017-04-04 05:51:49 -07001079 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001080 rtc::CritScope lock(&crit_);
1081 return min_transmit_bitrate_bps_;
1082 }
1083
Erik Språngd7329ca2019-02-21 21:19:53 +01001084 void SetNumExpectedLayers(size_t num_layers) {
1085 rtc::CritScope lock(&crit_);
1086 num_expected_layers_ = num_layers;
1087 }
1088
Erik Språngb7cb7b52019-02-26 15:52:33 +01001089 int64_t GetLastCaptureTimeMs() const {
1090 rtc::CritScope lock(&crit_);
1091 return last_capture_time_ms_;
1092 }
1093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001094 std::vector<uint8_t> GetLastEncodedImageData() {
1095 rtc::CritScope lock(&crit_);
1096 return std::move(last_encoded_image_data_);
1097 }
1098
1099 RTPFragmentationHeader GetLastFragmentation() {
1100 rtc::CritScope lock(&crit_);
1101 return std::move(last_fragmentation_);
1102 }
1103
perkj26091b12016-09-01 01:17:40 -07001104 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001105 Result OnEncodedImage(
1106 const EncodedImage& encoded_image,
1107 const CodecSpecificInfo* codec_specific_info,
1108 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001109 rtc::CritScope lock(&crit_);
1110 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001111 last_encoded_image_data_ = std::vector<uint8_t>(
1112 encoded_image.data(), encoded_image.data() + encoded_image.size());
1113 if (fragmentation) {
1114 last_fragmentation_.CopyFrom(*fragmentation);
1115 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001116 uint32_t timestamp = encoded_image.Timestamp();
1117 if (last_timestamp_ != timestamp) {
1118 num_received_layers_ = 1;
1119 } else {
1120 ++num_received_layers_;
1121 }
1122 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001123 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001124 last_width_ = encoded_image._encodedWidth;
1125 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001126 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001127 if (num_received_layers_ == num_expected_layers_) {
1128 encoded_frame_event_.Set();
1129 }
sprangb1ca0732017-02-01 08:38:12 -08001130 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001131 }
1132
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001133 void OnEncoderConfigurationChanged(
1134 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001135 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001136 VideoEncoderConfig::ContentType content_type,
1137 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001138 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001139 ++number_of_reconfigurations_;
1140 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1141 }
1142
perkj26091b12016-09-01 01:17:40 -07001143 rtc::CriticalSection crit_;
1144 TestEncoder* test_encoder_;
1145 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001146 std::vector<uint8_t> last_encoded_image_data_;
1147 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001148 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001149 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001150 uint32_t last_height_ = 0;
1151 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001152 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001153 size_t num_expected_layers_ = 1;
1154 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001155 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001156 int number_of_reconfigurations_ = 0;
1157 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001158 };
1159
Sergey Silkin5ee69672019-07-02 14:18:34 +02001160 class VideoBitrateAllocatorProxyFactory
1161 : public VideoBitrateAllocatorFactory {
1162 public:
1163 VideoBitrateAllocatorProxyFactory()
1164 : bitrate_allocator_factory_(
1165 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1166
1167 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1168 const VideoCodec& codec) override {
1169 rtc::CritScope lock(&crit_);
1170 codec_config_ = codec;
1171 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1172 }
1173
1174 VideoCodec codec_config() const {
1175 rtc::CritScope lock(&crit_);
1176 return codec_config_;
1177 }
1178
1179 private:
1180 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1181
1182 rtc::CriticalSection crit_;
1183 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1184 };
1185
perkj26091b12016-09-01 01:17:40 -07001186 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001187 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001188 int codec_width_;
1189 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001190 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001191 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001192 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001193 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001194 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001195 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001196 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001197 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001198 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001199 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001200};
1201
mflodmancc3d4422017-08-03 08:27:51 -07001202TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001203 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001204 DataRate::BitsPerSec(kTargetBitrateBps),
1205 DataRate::BitsPerSec(kTargetBitrateBps),
1206 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001207 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001208 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001209 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001210 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001211 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001212}
1213
mflodmancc3d4422017-08-03 08:27:51 -07001214TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001215 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001216 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001217 // The encoder will cache up to one frame for a short duration. Adding two
1218 // frames means that the first frame will be dropped and the second frame will
1219 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001220 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001221 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001222 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001223
Erik Språng4c6ca302019-04-08 15:14:01 +02001224 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001225 DataRate::BitsPerSec(kTargetBitrateBps),
1226 DataRate::BitsPerSec(kTargetBitrateBps),
1227 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001228
Sebastian Janssona3177052018-04-10 13:05:49 +02001229 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001230 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001231 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1232
1233 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001234 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001235}
1236
mflodmancc3d4422017-08-03 08:27:51 -07001237TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001238 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001239 DataRate::BitsPerSec(kTargetBitrateBps),
1240 DataRate::BitsPerSec(kTargetBitrateBps),
1241 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001242 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001243 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001244
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001245 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
1246 DataRate::BitsPerSec(0),
1247 DataRate::BitsPerSec(0), 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001248 // The encoder will cache up to one frame for a short duration. Adding two
1249 // frames means that the first frame will be dropped and the second frame will
1250 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001251 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001252 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001253
Erik Språng4c6ca302019-04-08 15:14:01 +02001254 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001255 DataRate::BitsPerSec(kTargetBitrateBps),
1256 DataRate::BitsPerSec(kTargetBitrateBps),
1257 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001258 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001259 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1260 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001261 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001262}
1263
mflodmancc3d4422017-08-03 08:27:51 -07001264TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001265 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001266 DataRate::BitsPerSec(kTargetBitrateBps),
1267 DataRate::BitsPerSec(kTargetBitrateBps),
1268 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001269 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001270 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001271
1272 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001273 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001274
perkja49cbd32016-09-16 07:53:41 -07001275 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001276 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001277 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001278}
1279
mflodmancc3d4422017-08-03 08:27:51 -07001280TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001281 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001282 DataRate::BitsPerSec(kTargetBitrateBps),
1283 DataRate::BitsPerSec(kTargetBitrateBps),
1284 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001285
perkja49cbd32016-09-16 07:53:41 -07001286 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001287 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001288
mflodmancc3d4422017-08-03 08:27:51 -07001289 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001290 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001291 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001292 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1293 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001294}
1295
mflodmancc3d4422017-08-03 08:27:51 -07001296TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001297 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001298 DataRate::BitsPerSec(kTargetBitrateBps),
1299 DataRate::BitsPerSec(kTargetBitrateBps),
1300 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001301
1302 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001303 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001304 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001305 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1306 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001307 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1308 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001309 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001310 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001311
mflodmancc3d4422017-08-03 08:27:51 -07001312 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001313}
1314
Noah Richards51db4212019-06-12 06:59:12 -07001315TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1316 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001317 DataRate::BitsPerSec(kTargetBitrateBps),
1318 DataRate::BitsPerSec(kTargetBitrateBps),
1319 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001320
1321 rtc::Event frame_destroyed_event;
1322 video_source_.IncomingCapturedFrame(
1323 CreateFakeNativeFrame(1, &frame_destroyed_event));
1324 ExpectDroppedFrame();
1325 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1326 video_stream_encoder_->Stop();
1327}
1328
1329TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1330 // Use the cropping factory.
1331 video_encoder_config_.video_stream_factory =
1332 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1333 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1334 kMaxPayloadLength);
1335 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1336
1337 // Capture a frame at codec_width_/codec_height_.
1338 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001339 DataRate::BitsPerSec(kTargetBitrateBps),
1340 DataRate::BitsPerSec(kTargetBitrateBps),
1341 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001342 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1343 WaitForEncodedFrame(1);
1344 // The encoder will have been configured once.
1345 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1346 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1347 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1348
1349 // Now send in a fake frame that needs to be cropped as the width/height
1350 // aren't divisible by 4 (see CreateEncoderStreams above).
1351 rtc::Event frame_destroyed_event;
1352 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1353 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1354 ExpectDroppedFrame();
1355 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1356 video_stream_encoder_->Stop();
1357}
1358
Ying Wang9b881ab2020-02-07 14:29:32 +01001359TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
1360 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001361 DataRate::BitsPerSec(kTargetBitrateBps),
1362 DataRate::BitsPerSec(kTargetBitrateBps),
1363 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001364 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1365 WaitForEncodedFrame(1);
1366
1367 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001368 DataRate::BitsPerSec(kTargetBitrateBps),
1369 DataRate::BitsPerSec(kTargetBitrateBps),
1370 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001371 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1372 // frames. Adding two frames means that the first frame will be dropped and
1373 // the second frame will be sent to the encoder.
1374 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1375 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1376 WaitForEncodedFrame(3);
1377 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1378 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1379 WaitForEncodedFrame(5);
1380 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1381 video_stream_encoder_->Stop();
1382}
1383
mflodmancc3d4422017-08-03 08:27:51 -07001384TEST_F(VideoStreamEncoderTest,
1385 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001386 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001387 DataRate::BitsPerSec(kTargetBitrateBps),
1388 DataRate::BitsPerSec(kTargetBitrateBps),
1389 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001390 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001391
1392 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001393 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001394 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001395 // The encoder will have been configured once when the first frame is
1396 // received.
1397 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001398
1399 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001400 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001401 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001402 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001403 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001404
1405 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001406 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001407 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001408 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001409 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001410
mflodmancc3d4422017-08-03 08:27:51 -07001411 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001412}
1413
mflodmancc3d4422017-08-03 08:27:51 -07001414TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001415 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001416 DataRate::BitsPerSec(kTargetBitrateBps),
1417 DataRate::BitsPerSec(kTargetBitrateBps),
1418 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001419
1420 // Capture a frame and wait for it to synchronize with the encoder thread.
1421 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001422 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001423 // The encoder will have been configured once.
1424 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001425 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1426 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1427
1428 codec_width_ *= 2;
1429 codec_height_ *= 2;
1430 // Capture a frame with a higher resolution and wait for it to synchronize
1431 // with the encoder thread.
1432 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001433 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001434 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1435 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001436 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001437
mflodmancc3d4422017-08-03 08:27:51 -07001438 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001439}
1440
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001441TEST_F(VideoStreamEncoderTest,
1442 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1443 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001444 DataRate::BitsPerSec(kTargetBitrateBps),
1445 DataRate::BitsPerSec(kTargetBitrateBps),
1446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001447
1448 // Capture a frame and wait for it to synchronize with the encoder thread.
1449 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1450 WaitForEncodedFrame(1);
1451
1452 VideoEncoderConfig video_encoder_config;
1453 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1454 // Changing the max payload data length recreates encoder.
1455 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1456 kMaxPayloadLength / 2);
1457
1458 // Capture a frame and wait for it to synchronize with the encoder thread.
1459 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1460 WaitForEncodedFrame(2);
1461 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1462
1463 video_stream_encoder_->Stop();
1464}
1465
Sergey Silkin5ee69672019-07-02 14:18:34 +02001466TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1467 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001468 DataRate::BitsPerSec(kTargetBitrateBps),
1469 DataRate::BitsPerSec(kTargetBitrateBps),
1470 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001471
1472 VideoEncoderConfig video_encoder_config;
1473 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1474 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1475 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1476 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1477 kMaxPayloadLength);
1478
1479 // Capture a frame and wait for it to synchronize with the encoder thread.
1480 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1481 WaitForEncodedFrame(1);
1482 // The encoder will have been configured once when the first frame is
1483 // received.
1484 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1485 EXPECT_EQ(kTargetBitrateBps,
1486 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1487 EXPECT_EQ(kStartBitrateBps,
1488 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1489
Sergey Silkin6456e352019-07-08 17:56:40 +02001490 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1491 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001492 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1493 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1494 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1495 kMaxPayloadLength);
1496
1497 // Capture a frame and wait for it to synchronize with the encoder thread.
1498 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1499 WaitForEncodedFrame(2);
1500 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1501 // Bitrate limits have changed - rate allocator should be reconfigured,
1502 // encoder should not be reconfigured.
1503 EXPECT_EQ(kTargetBitrateBps * 2,
1504 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1505 EXPECT_EQ(kStartBitrateBps * 2,
1506 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1507 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1508
1509 video_stream_encoder_->Stop();
1510}
1511
Sergey Silkin6456e352019-07-08 17:56:40 +02001512TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001513 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001514 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001515 DataRate::BitsPerSec(kTargetBitrateBps),
1516 DataRate::BitsPerSec(kTargetBitrateBps),
1517 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001518
Sergey Silkincd02eba2020-01-20 14:48:40 +01001519 const uint32_t kMinEncBitrateKbps = 100;
1520 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001521 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001522 /*frame_size_pixels=*/codec_width_ * codec_height_,
1523 /*min_start_bitrate_bps=*/0,
1524 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1525 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001526 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1527
Sergey Silkincd02eba2020-01-20 14:48:40 +01001528 VideoEncoderConfig video_encoder_config;
1529 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1530 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1531 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1532 (kMinEncBitrateKbps + 1) * 1000;
1533 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1534 kMaxPayloadLength);
1535
1536 // When both encoder and app provide bitrate limits, the intersection of
1537 // provided sets should be used.
1538 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1539 WaitForEncodedFrame(1);
1540 EXPECT_EQ(kMaxEncBitrateKbps,
1541 bitrate_allocator_factory_.codec_config().maxBitrate);
1542 EXPECT_EQ(kMinEncBitrateKbps + 1,
1543 bitrate_allocator_factory_.codec_config().minBitrate);
1544
1545 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1546 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1547 (kMinEncBitrateKbps - 1) * 1000;
1548 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1549 kMaxPayloadLength);
1550 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001551 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001552 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001553 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001554 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001555 bitrate_allocator_factory_.codec_config().minBitrate);
1556
Sergey Silkincd02eba2020-01-20 14:48:40 +01001557 video_stream_encoder_->Stop();
1558}
1559
1560TEST_F(VideoStreamEncoderTest,
1561 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
1562 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001563 DataRate::BitsPerSec(kTargetBitrateBps),
1564 DataRate::BitsPerSec(kTargetBitrateBps),
1565 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001566
1567 const uint32_t kMinAppBitrateKbps = 100;
1568 const uint32_t kMaxAppBitrateKbps = 200;
1569 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1570 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1571 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1572 /*frame_size_pixels=*/codec_width_ * codec_height_,
1573 /*min_start_bitrate_bps=*/0,
1574 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1575 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1576 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1577
1578 VideoEncoderConfig video_encoder_config;
1579 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1580 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1581 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1582 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001583 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1584 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001585
Sergey Silkincd02eba2020-01-20 14:48:40 +01001586 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1587 WaitForEncodedFrame(1);
1588 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001589 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001590 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001591 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001592
1593 video_stream_encoder_->Stop();
1594}
1595
1596TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001597 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001598 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001599 DataRate::BitsPerSec(kTargetBitrateBps),
1600 DataRate::BitsPerSec(kTargetBitrateBps),
1601 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001602
1603 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001604 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001605 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001606 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001607 fake_encoder_.SetResolutionBitrateLimits(
1608 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1609
1610 VideoEncoderConfig video_encoder_config;
1611 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1612 video_encoder_config.max_bitrate_bps = 0;
1613 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1614 kMaxPayloadLength);
1615
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001616 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001617 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1618 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001619 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1620 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001621 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1622 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1623
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001624 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001625 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1626 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001627 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1628 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001629 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1630 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1631
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001632 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001633 // encoder for 360p should be used.
1634 video_source_.IncomingCapturedFrame(
1635 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1636 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001637 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1638 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001639 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1640 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1641
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001642 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001643 // ignored.
1644 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1645 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001646 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1647 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001648 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1649 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001650 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1651 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001652 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1653 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1654
1655 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1656 // for 270p should be used.
1657 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1658 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001659 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1660 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001661 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1662 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1663
1664 video_stream_encoder_->Stop();
1665}
1666
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001667TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1668 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001669 DataRate::BitsPerSec(kTargetBitrateBps),
1670 DataRate::BitsPerSec(kTargetBitrateBps),
1671 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001672
1673 VideoEncoderConfig video_encoder_config;
1674 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1675 video_encoder_config.max_bitrate_bps = 0;
1676 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1677 kMaxPayloadLength);
1678
1679 // Encode 720p frame to get the default encoder target bitrate.
1680 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1681 WaitForEncodedFrame(1);
1682 const uint32_t kDefaultTargetBitrateFor720pKbps =
1683 bitrate_allocator_factory_.codec_config()
1684 .simulcastStream[0]
1685 .targetBitrate;
1686
1687 // Set the max recommended encoder bitrate to something lower than the default
1688 // target bitrate.
1689 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1690 1280 * 720, 10 * 1000, 10 * 1000,
1691 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1692 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1693
1694 // Change resolution to trigger encoder reinitialization.
1695 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1696 WaitForEncodedFrame(2);
1697 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1698 WaitForEncodedFrame(3);
1699
1700 // Ensure the target bitrate is capped by the max bitrate.
1701 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1702 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1703 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1704 .simulcastStream[0]
1705 .targetBitrate *
1706 1000,
1707 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1708
1709 video_stream_encoder_->Stop();
1710}
1711
mflodmancc3d4422017-08-03 08:27:51 -07001712TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001713 EXPECT_TRUE(video_source_.has_sinks());
1714 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001715 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001716 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001717 EXPECT_FALSE(video_source_.has_sinks());
1718 EXPECT_TRUE(new_video_source.has_sinks());
1719
mflodmancc3d4422017-08-03 08:27:51 -07001720 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001721}
1722
mflodmancc3d4422017-08-03 08:27:51 -07001723TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001724 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001725 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001726 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001727 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001728}
1729
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001730TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1731 constexpr int kRequestedResolutionAlignment = 7;
1732 video_source_.set_adaptation_enabled(true);
1733 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
1734 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001735 DataRate::BitsPerSec(kTargetBitrateBps),
1736 DataRate::BitsPerSec(kTargetBitrateBps),
1737 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001738
1739 // On the 1st frame, we should have initialized the encoder and
1740 // asked for its resolution requirements.
1741 video_source_.IncomingCapturedFrame(
1742 CreateFrame(1, codec_width_, codec_height_));
1743 WaitForEncodedFrame(1);
1744 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1745 kRequestedResolutionAlignment);
1746
1747 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1748 // (It's up the to the encoder to potentially drop the previous frame,
1749 // to avoid coding back-to-back keyframes.)
1750 video_source_.IncomingCapturedFrame(
1751 CreateFrame(2, codec_width_, codec_height_));
1752 WaitForEncodedFrame(2);
1753 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1754
1755 video_stream_encoder_->Stop();
1756}
1757
Jonathan Yubc771b72017-12-08 17:04:29 -08001758TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1759 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001760 const int kWidth = 1280;
1761 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001762
1763 // We rely on the automatic resolution adaptation, but we handle framerate
1764 // adaptation manually by mocking the stats proxy.
1765 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001766
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001767 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001768 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps),
1771 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001772 video_stream_encoder_->SetSource(&video_source_,
1773 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001774 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001775 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001776 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001777 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1778
Jonathan Yubc771b72017-12-08 17:04:29 -08001779 // Adapt down as far as possible.
1780 rtc::VideoSinkWants last_wants;
1781 int64_t t = 1;
1782 int loop_count = 0;
1783 do {
1784 ++loop_count;
1785 last_wants = video_source_.sink_wants();
1786
1787 // Simulate the framerate we've been asked to adapt to.
1788 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1789 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1790 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1791 mock_stats.input_frame_rate = fps;
1792 stats_proxy_->SetMockStats(mock_stats);
1793
1794 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1795 sink_.WaitForEncodedFrame(t);
1796 t += frame_interval_ms;
1797
mflodmancc3d4422017-08-03 08:27:51 -07001798 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001799 VerifyBalancedModeFpsRange(
1800 video_source_.sink_wants(),
1801 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1802 } while (video_source_.sink_wants().max_pixel_count <
1803 last_wants.max_pixel_count ||
1804 video_source_.sink_wants().max_framerate_fps <
1805 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001806
Jonathan Yubc771b72017-12-08 17:04:29 -08001807 // Verify that we've adapted all the way down.
1808 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001809 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001810 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1811 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001812 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001813 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1814 *video_source_.last_sent_height());
1815 EXPECT_EQ(kMinBalancedFramerateFps,
1816 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001817
Jonathan Yubc771b72017-12-08 17:04:29 -08001818 // Adapt back up the same number of times we adapted down.
1819 for (int i = 0; i < loop_count - 1; ++i) {
1820 last_wants = video_source_.sink_wants();
1821
1822 // Simulate the framerate we've been asked to adapt to.
1823 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1824 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1825 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1826 mock_stats.input_frame_rate = fps;
1827 stats_proxy_->SetMockStats(mock_stats);
1828
1829 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1830 sink_.WaitForEncodedFrame(t);
1831 t += frame_interval_ms;
1832
mflodmancc3d4422017-08-03 08:27:51 -07001833 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001834 VerifyBalancedModeFpsRange(
1835 video_source_.sink_wants(),
1836 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1837 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1838 last_wants.max_pixel_count ||
1839 video_source_.sink_wants().max_framerate_fps >
1840 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001841 }
1842
Åsa Persson8c1bf952018-09-13 10:42:19 +02001843 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001844 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001846 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1847 EXPECT_EQ((loop_count - 1) * 2,
1848 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001849
mflodmancc3d4422017-08-03 08:27:51 -07001850 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001851}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001852
mflodmancc3d4422017-08-03 08:27:51 -07001853TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001854 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001855 DataRate::BitsPerSec(kTargetBitrateBps),
1856 DataRate::BitsPerSec(kTargetBitrateBps),
1857 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001858 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001859
sprangc5d62e22017-04-02 23:53:04 -07001860 const int kFrameWidth = 1280;
1861 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001862
Åsa Persson8c1bf952018-09-13 10:42:19 +02001863 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001864
kthelgason5e13d412016-12-01 03:59:51 -08001865 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001866 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001867 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001868 frame_timestamp += kFrameIntervalMs;
1869
perkj803d97f2016-11-01 11:45:46 -07001870 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001871 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001872 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001873 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001874 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001875 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001876
asapersson0944a802017-04-07 00:57:58 -07001877 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001878 // wanted resolution.
1879 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1880 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1881 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001882 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001883
1884 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001885 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001886 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001887 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001888 // Give the encoder queue time to process the change in degradation preference
1889 // by waiting for an encoded frame.
1890 new_video_source.IncomingCapturedFrame(
1891 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1892 sink_.WaitForEncodedFrame(frame_timestamp);
1893 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001894 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001895 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001896
sprangc5d62e22017-04-02 23:53:04 -07001897 // Force an input frame rate to be available, or the adaptation call won't
1898 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001899 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001900 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001901 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001902 stats_proxy_->SetMockStats(stats);
1903
mflodmancc3d4422017-08-03 08:27:51 -07001904 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001905 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001906 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001907 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001908 frame_timestamp += kFrameIntervalMs;
1909
1910 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001911 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001912 EXPECT_EQ(std::numeric_limits<int>::max(),
1913 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001914 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001915
asapersson02465b82017-04-10 01:12:52 -07001916 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001917 video_stream_encoder_->SetSource(&new_video_source,
1918 webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01001919 // Give the encoder queue time to process the change in degradation preference
1920 // by waiting for an encoded frame.
1921 new_video_source.IncomingCapturedFrame(
1922 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1923 sink_.WaitForEncodedFrame(frame_timestamp);
1924 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001925 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001926
mflodmancc3d4422017-08-03 08:27:51 -07001927 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001928 new_video_source.IncomingCapturedFrame(
1929 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001930 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001931 frame_timestamp += kFrameIntervalMs;
1932
1933 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001934 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001935
1936 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001937 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001938 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01001939 // Give the encoder queue time to process the change in degradation preference
1940 // by waiting for an encoded frame.
1941 new_video_source.IncomingCapturedFrame(
1942 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1943 sink_.WaitForEncodedFrame(frame_timestamp);
1944 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001945 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1946 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001947 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001948 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001949
1950 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001951 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001952 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001953 // Give the encoder queue time to process the change in degradation preference
1954 // by waiting for an encoded frame.
1955 new_video_source.IncomingCapturedFrame(
1956 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1957 sink_.WaitForEncodedFrame(frame_timestamp);
1958 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001959 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1960 EXPECT_EQ(std::numeric_limits<int>::max(),
1961 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001962 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001963
mflodmancc3d4422017-08-03 08:27:51 -07001964 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001965}
1966
mflodmancc3d4422017-08-03 08:27:51 -07001967TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001968 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001969 DataRate::BitsPerSec(kTargetBitrateBps),
1970 DataRate::BitsPerSec(kTargetBitrateBps),
1971 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001972
asaperssonfab67072017-04-04 05:51:49 -07001973 const int kWidth = 1280;
1974 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001975 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001976 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001977 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1978 EXPECT_FALSE(stats.bw_limited_resolution);
1979 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1980
1981 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001982 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001983 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001984 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001985
1986 stats = stats_proxy_->GetStats();
1987 EXPECT_TRUE(stats.bw_limited_resolution);
1988 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1989
1990 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001991 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07001992 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001993 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07001994
1995 stats = stats_proxy_->GetStats();
1996 EXPECT_FALSE(stats.bw_limited_resolution);
1997 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
1998 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
1999
mflodmancc3d4422017-08-03 08:27:51 -07002000 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002001}
2002
mflodmancc3d4422017-08-03 08:27:51 -07002003TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002004 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002005 DataRate::BitsPerSec(kTargetBitrateBps),
2006 DataRate::BitsPerSec(kTargetBitrateBps),
2007 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002008
2009 const int kWidth = 1280;
2010 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002011 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002012 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002013 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2014 EXPECT_FALSE(stats.cpu_limited_resolution);
2015 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2016
2017 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002018 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002019 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002020 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002021
2022 stats = stats_proxy_->GetStats();
2023 EXPECT_TRUE(stats.cpu_limited_resolution);
2024 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2025
2026 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07002027 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002028 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002029 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002030
2031 stats = stats_proxy_->GetStats();
2032 EXPECT_FALSE(stats.cpu_limited_resolution);
2033 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002034 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002035
mflodmancc3d4422017-08-03 08:27:51 -07002036 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002037}
2038
mflodmancc3d4422017-08-03 08:27:51 -07002039TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002040 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002041 DataRate::BitsPerSec(kTargetBitrateBps),
2042 DataRate::BitsPerSec(kTargetBitrateBps),
2043 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002044
asaperssonfab67072017-04-04 05:51:49 -07002045 const int kWidth = 1280;
2046 const int kHeight = 720;
2047 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002048 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002049 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002050 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002051 EXPECT_FALSE(stats.cpu_limited_resolution);
2052 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2053
asaperssonfab67072017-04-04 05:51:49 -07002054 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002055 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002056 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002057 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002058 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002059 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002060 EXPECT_TRUE(stats.cpu_limited_resolution);
2061 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2062
2063 // Set new source with adaptation still enabled.
2064 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002065 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002066 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002067
asaperssonfab67072017-04-04 05:51:49 -07002068 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002069 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002070 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002071 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002072 EXPECT_TRUE(stats.cpu_limited_resolution);
2073 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2074
2075 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002076 video_stream_encoder_->SetSource(&new_video_source,
2077 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002078
asaperssonfab67072017-04-04 05:51:49 -07002079 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002080 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002081 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002082 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002083 EXPECT_FALSE(stats.cpu_limited_resolution);
2084 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2085
2086 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002087 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002088 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002089
asaperssonfab67072017-04-04 05:51:49 -07002090 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002091 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002092 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002093 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002094 EXPECT_TRUE(stats.cpu_limited_resolution);
2095 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2096
asaperssonfab67072017-04-04 05:51:49 -07002097 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07002098 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002099 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002100 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002101 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002102 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002103 EXPECT_FALSE(stats.cpu_limited_resolution);
2104 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002105 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002106
mflodmancc3d4422017-08-03 08:27:51 -07002107 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002108}
2109
mflodmancc3d4422017-08-03 08:27:51 -07002110TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002111 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002112 DataRate::BitsPerSec(kTargetBitrateBps),
2113 DataRate::BitsPerSec(kTargetBitrateBps),
2114 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002115
asaperssonfab67072017-04-04 05:51:49 -07002116 const int kWidth = 1280;
2117 const int kHeight = 720;
2118 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002119 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002120 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002121 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002122 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002123 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002124
2125 // Set new source with adaptation still enabled.
2126 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002127 video_stream_encoder_->SetSource(&new_video_source,
2128 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002129
asaperssonfab67072017-04-04 05:51:49 -07002130 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002131 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002132 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002133 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002134 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002135 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002136
asaperssonfab67072017-04-04 05:51:49 -07002137 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002138 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002139 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002140 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002141 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002142 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002143 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002144 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002145
asaperssonfab67072017-04-04 05:51:49 -07002146 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002147 video_stream_encoder_->SetSource(&new_video_source,
2148 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002149
asaperssonfab67072017-04-04 05:51:49 -07002150 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002151 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002152 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002153 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002154 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002155 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002156
asapersson02465b82017-04-10 01:12:52 -07002157 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002158 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002159 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002160
asaperssonfab67072017-04-04 05:51:49 -07002161 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002162 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002163 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002164 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002165 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002166 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2167 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002168
mflodmancc3d4422017-08-03 08:27:51 -07002169 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002170}
2171
mflodmancc3d4422017-08-03 08:27:51 -07002172TEST_F(VideoStreamEncoderTest,
2173 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002174 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002175 DataRate::BitsPerSec(kTargetBitrateBps),
2176 DataRate::BitsPerSec(kTargetBitrateBps),
2177 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002178
2179 const int kWidth = 1280;
2180 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002181 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002182 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002183 video_source_.IncomingCapturedFrame(
2184 CreateFrame(timestamp_ms, kWidth, kHeight));
2185 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002186 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2187 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2188 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2189
2190 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002191 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002192 timestamp_ms += kFrameIntervalMs;
2193 video_source_.IncomingCapturedFrame(
2194 CreateFrame(timestamp_ms, kWidth, kHeight));
2195 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002196 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2197 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2198 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2199
2200 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002201 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002202 timestamp_ms += kFrameIntervalMs;
2203 video_source_.IncomingCapturedFrame(
2204 CreateFrame(timestamp_ms, kWidth, kHeight));
2205 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002206 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2207 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2208 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2209
Niels Möller4db138e2018-04-19 09:04:13 +02002210 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002211 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002212
2213 VideoEncoderConfig video_encoder_config;
2214 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2215 // Make format different, to force recreation of encoder.
2216 video_encoder_config.video_format.parameters["foo"] = "foo";
2217 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002218 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002219 timestamp_ms += kFrameIntervalMs;
2220 video_source_.IncomingCapturedFrame(
2221 CreateFrame(timestamp_ms, kWidth, kHeight));
2222 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002223 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2224 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2225 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2226
mflodmancc3d4422017-08-03 08:27:51 -07002227 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002228}
2229
mflodmancc3d4422017-08-03 08:27:51 -07002230TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002231 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
2232 video_stream_encoder_->OnBitrateUpdated(
2233 DataRate::BitsPerSec(kTargetBitrateBps),
2234 DataRate::BitsPerSec(kTargetBitrateBps),
2235 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2236
2237 const int kWidth = 1280;
2238 const int kHeight = 720;
2239 int sequence = 1;
2240
2241 // Enable BALANCED preference, no initial limitation.
2242 test::FrameForwarder source;
2243 video_stream_encoder_->SetSource(&source,
2244 webrtc::DegradationPreference::BALANCED);
2245 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2246 WaitForEncodedFrame(sequence++);
2247 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2248 EXPECT_FALSE(stats.cpu_limited_resolution);
2249 EXPECT_FALSE(stats.cpu_limited_framerate);
2250 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2251
2252 // Trigger CPU overuse, should now adapt down.
2253 video_stream_encoder_->TriggerCpuOveruse();
2254 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2255 WaitForEncodedFrame(sequence++);
2256 stats = stats_proxy_->GetStats();
2257 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2258
2259 // Set new degradation preference should clear restrictions since we changed
2260 // from BALANCED.
2261 video_stream_encoder_->SetSource(
2262 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2263 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2264 WaitForEncodedFrame(sequence++);
2265 stats = stats_proxy_->GetStats();
2266 EXPECT_FALSE(stats.cpu_limited_resolution);
2267 EXPECT_FALSE(stats.cpu_limited_framerate);
2268 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2269
2270 // Force an input frame rate to be available, or the adaptation call won't
2271 // know what framerate to adapt from.
2272 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2273 mock_stats.input_frame_rate = 30;
2274 stats_proxy_->SetMockStats(mock_stats);
2275 video_stream_encoder_->TriggerCpuOveruse();
2276 stats_proxy_->ResetMockStats();
2277 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2278 WaitForEncodedFrame(sequence++);
2279
2280 // We have now adapted once.
2281 stats = stats_proxy_->GetStats();
2282 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2283
2284 // Back to BALANCED, should clear the restrictions again.
2285 video_stream_encoder_->SetSource(&source,
2286 webrtc::DegradationPreference::BALANCED);
2287 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2288 WaitForEncodedFrame(sequence++);
2289 stats = stats_proxy_->GetStats();
2290 EXPECT_FALSE(stats.cpu_limited_resolution);
2291 EXPECT_FALSE(stats.cpu_limited_framerate);
2292 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2293
2294 video_stream_encoder_->Stop();
2295}
2296
2297TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002298 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002299 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002300 DataRate::BitsPerSec(kTargetBitrateBps),
2301 DataRate::BitsPerSec(kTargetBitrateBps),
2302 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002303
asapersson0944a802017-04-07 00:57:58 -07002304 const int kWidth = 1280;
2305 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002306 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002307
asaperssonfab67072017-04-04 05:51:49 -07002308 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002309 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002310 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002311 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002312 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002313 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2314
asapersson02465b82017-04-10 01:12:52 -07002315 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002316 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002317 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002318 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002319 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002320 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002321 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002322 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2323
2324 // Set new source with adaptation still enabled.
2325 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002326 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002327 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002328
2329 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002330 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002331 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002332 stats = stats_proxy_->GetStats();
2333 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002334 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002335 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2336
sprangc5d62e22017-04-02 23:53:04 -07002337 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002338 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002339 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002340 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002341 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002342 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002343 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002344 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002345 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002346 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002347 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2348
sprangc5d62e22017-04-02 23:53:04 -07002349 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002350 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002351 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2352 mock_stats.input_frame_rate = 30;
2353 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002354 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002355 stats_proxy_->ResetMockStats();
2356
2357 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002358 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002359 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002360
2361 // Framerate now adapted.
2362 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002363 EXPECT_FALSE(stats.cpu_limited_resolution);
2364 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002365 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2366
2367 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002368 video_stream_encoder_->SetSource(&new_video_source,
2369 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002370 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002371 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002372 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002373
2374 stats = stats_proxy_->GetStats();
2375 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002376 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002377 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2378
2379 // Try to trigger overuse. Should not succeed.
2380 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002381 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002382 stats_proxy_->ResetMockStats();
2383
2384 stats = stats_proxy_->GetStats();
2385 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002386 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002387 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2388
2389 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002390 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002391 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002392 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002393 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002394 stats = stats_proxy_->GetStats();
2395 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002396 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002397 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002398
2399 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002400 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002401 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002402 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002403 stats = stats_proxy_->GetStats();
2404 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002405 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002406 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2407
2408 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002409 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002410 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002411 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002412 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002413 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002414 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002415 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002416 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002417 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002418 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2419
2420 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002421 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002422 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002423 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002424 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002425 stats = stats_proxy_->GetStats();
2426 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002427 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002428 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002429 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002430
mflodmancc3d4422017-08-03 08:27:51 -07002431 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002432}
2433
mflodmancc3d4422017-08-03 08:27:51 -07002434TEST_F(VideoStreamEncoderTest,
2435 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002436 const int kWidth = 1280;
2437 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002438 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002439 DataRate::BitsPerSec(kTargetBitrateBps),
2440 DataRate::BitsPerSec(kTargetBitrateBps),
2441 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002442
asaperssonfab67072017-04-04 05:51:49 -07002443 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002444 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002445
asaperssonfab67072017-04-04 05:51:49 -07002446 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002447 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002448
asaperssonfab67072017-04-04 05:51:49 -07002449 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002450 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002451
asaperssonfab67072017-04-04 05:51:49 -07002452 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002453 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002454
kthelgason876222f2016-11-29 01:44:11 -08002455 // Expect a scale down.
2456 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002457 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002458
asapersson02465b82017-04-10 01:12:52 -07002459 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002460 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002461 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002462 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002463
asaperssonfab67072017-04-04 05:51:49 -07002464 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002465 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002466 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002467 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002468
asaperssonfab67072017-04-04 05:51:49 -07002469 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002470 EXPECT_EQ(std::numeric_limits<int>::max(),
2471 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002472
asaperssonfab67072017-04-04 05:51:49 -07002473 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002474 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002475 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002476 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002477
asapersson02465b82017-04-10 01:12:52 -07002478 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002479 EXPECT_EQ(std::numeric_limits<int>::max(),
2480 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002481
mflodmancc3d4422017-08-03 08:27:51 -07002482 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002483}
2484
mflodmancc3d4422017-08-03 08:27:51 -07002485TEST_F(VideoStreamEncoderTest,
2486 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002487 const int kWidth = 1280;
2488 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002489 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002490 DataRate::BitsPerSec(kTargetBitrateBps),
2491 DataRate::BitsPerSec(kTargetBitrateBps),
2492 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002493
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002494 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002495 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002496 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002497 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002498
2499 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002500 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002501 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002502 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2503 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2504
2505 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002506 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002507 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002508 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2509 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2510 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2511
2512 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002513 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002514 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2515 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2516 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2517
mflodmancc3d4422017-08-03 08:27:51 -07002518 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002519}
2520
mflodmancc3d4422017-08-03 08:27:51 -07002521TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002522 const int kWidth = 1280;
2523 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002524 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002525 DataRate::BitsPerSec(kTargetBitrateBps),
2526 DataRate::BitsPerSec(kTargetBitrateBps),
2527 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002528
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002529 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002530 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002531 video_stream_encoder_->SetSource(&source,
2532 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002533 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2534 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002535 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002536
2537 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002538 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002539 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2540 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2541 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2542 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2543
2544 // Trigger adapt down for same input resolution, expect no change.
2545 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2546 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002547 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002548 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2550 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2551
2552 // Trigger adapt down for larger input resolution, expect no change.
2553 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2554 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002555 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002556 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2558 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2559
mflodmancc3d4422017-08-03 08:27:51 -07002560 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002561}
2562
mflodmancc3d4422017-08-03 08:27:51 -07002563TEST_F(VideoStreamEncoderTest,
2564 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002565 const int kWidth = 1280;
2566 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002567 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002568 DataRate::BitsPerSec(kTargetBitrateBps),
2569 DataRate::BitsPerSec(kTargetBitrateBps),
2570 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002571
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002572 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002573 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002574 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002575 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002576
2577 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002578 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002579 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002580 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2581 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2582
2583 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002584 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002585 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002586 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2587 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2588
mflodmancc3d4422017-08-03 08:27:51 -07002589 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002590}
2591
mflodmancc3d4422017-08-03 08:27:51 -07002592TEST_F(VideoStreamEncoderTest,
2593 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002594 const int kWidth = 1280;
2595 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002596 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002597 DataRate::BitsPerSec(kTargetBitrateBps),
2598 DataRate::BitsPerSec(kTargetBitrateBps),
2599 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002600
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002601 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002602 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002603 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002604 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002605
2606 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002607 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002608 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002609 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002610 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2611
2612 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002613 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002614 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002616 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2617
mflodmancc3d4422017-08-03 08:27:51 -07002618 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002619}
2620
mflodmancc3d4422017-08-03 08:27:51 -07002621TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002622 const int kWidth = 1280;
2623 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002624 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002625 DataRate::BitsPerSec(kTargetBitrateBps),
2626 DataRate::BitsPerSec(kTargetBitrateBps),
2627 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002628
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002629 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002630 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002631 video_stream_encoder_->SetSource(&source,
2632 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002633
2634 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2635 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002636 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002637 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2639 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2640
2641 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002642 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002643 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002644 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2645 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2646 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2647
mflodmancc3d4422017-08-03 08:27:51 -07002648 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002649}
2650
mflodmancc3d4422017-08-03 08:27:51 -07002651TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002652 const int kWidth = 1280;
2653 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002654 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002655 DataRate::BitsPerSec(kTargetBitrateBps),
2656 DataRate::BitsPerSec(kTargetBitrateBps),
2657 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002658
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002659 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002660 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002661 video_stream_encoder_->SetSource(&source,
2662 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002663
2664 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2665 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002666 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002667 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2668 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2669 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2670
2671 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002672 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002673 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2675 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2676 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2677
mflodmancc3d4422017-08-03 08:27:51 -07002678 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002679}
2680
mflodmancc3d4422017-08-03 08:27:51 -07002681TEST_F(VideoStreamEncoderTest,
2682 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002683 const int kWidth = 1280;
2684 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002685 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002686 DataRate::BitsPerSec(kTargetBitrateBps),
2687 DataRate::BitsPerSec(kTargetBitrateBps),
2688 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002689
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002690 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002691 AdaptingFrameForwarder source;
2692 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002693 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002694 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002695
2696 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002697 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002698 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002699 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2700 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2701
2702 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002703 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002704 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002705 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002706 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002707 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2708 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2709
2710 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002711 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002712 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002713 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2714 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2715 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2716
mflodmancc3d4422017-08-03 08:27:51 -07002717 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002718}
2719
mflodmancc3d4422017-08-03 08:27:51 -07002720TEST_F(VideoStreamEncoderTest,
2721 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002722 const int kWidth = 1280;
2723 const int kHeight = 720;
2724 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002725 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002726 DataRate::BitsPerSec(kTargetBitrateBps),
2727 DataRate::BitsPerSec(kTargetBitrateBps),
2728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002729
2730 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2731 stats.input_frame_rate = kInputFps;
2732 stats_proxy_->SetMockStats(stats);
2733
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002734 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002735 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2736 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002737 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002738
2739 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002740 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002741 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2742 sink_.WaitForEncodedFrame(2);
2743 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2744
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002745 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002746 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002747 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002748 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002749 // Give the encoder queue time to process the change in degradation preference
2750 // by waiting for an encoded frame.
2751 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2752 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002753 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002754
2755 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002756 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002757 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2758 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002759 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2760
2761 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002762 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002763 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002764
mflodmancc3d4422017-08-03 08:27:51 -07002765 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002766}
2767
mflodmancc3d4422017-08-03 08:27:51 -07002768TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002769 const int kWidth = 1280;
2770 const int kHeight = 720;
2771 const size_t kNumFrames = 10;
2772
Erik Språng4c6ca302019-04-08 15:14:01 +02002773 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002774 DataRate::BitsPerSec(kTargetBitrateBps),
2775 DataRate::BitsPerSec(kTargetBitrateBps),
2776 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002777
asaperssond0de2952017-04-21 01:47:31 -07002778 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002779 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002780 video_source_.set_adaptation_enabled(true);
2781
2782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2783 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2784
2785 int downscales = 0;
2786 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002787 video_source_.IncomingCapturedFrame(
2788 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2789 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002790
asaperssonfab67072017-04-04 05:51:49 -07002791 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002792 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002793 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002794 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002795
2796 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2797 ++downscales;
2798
2799 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2800 EXPECT_EQ(downscales,
2801 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2802 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002803 }
mflodmancc3d4422017-08-03 08:27:51 -07002804 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002805}
2806
mflodmancc3d4422017-08-03 08:27:51 -07002807TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002808 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2809 const int kWidth = 1280;
2810 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002811 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002812 DataRate::BitsPerSec(kTargetBitrateBps),
2813 DataRate::BitsPerSec(kTargetBitrateBps),
2814 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002815
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002816 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002817 AdaptingFrameForwarder source;
2818 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002820 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002821
Åsa Persson8c1bf952018-09-13 10:42:19 +02002822 int64_t timestamp_ms = kFrameIntervalMs;
2823 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002824 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002825 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002826 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2827 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2828
2829 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002830 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002831 timestamp_ms += kFrameIntervalMs;
2832 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2833 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002834 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002835 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2836 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2837
2838 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002839 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002840 timestamp_ms += kFrameIntervalMs;
2841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002842 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002843 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2845 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2846
2847 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002848 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002849 timestamp_ms += kFrameIntervalMs;
2850 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2851 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002852 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002853 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2854 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2855
2856 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002858 timestamp_ms += kFrameIntervalMs;
2859 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002860 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002861 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002862 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2863 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2864
mflodmancc3d4422017-08-03 08:27:51 -07002865 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002866}
2867
mflodmancc3d4422017-08-03 08:27:51 -07002868TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002869 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2870 const int kWidth = 1280;
2871 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002872 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002873 DataRate::BitsPerSec(kTargetBitrateBps),
2874 DataRate::BitsPerSec(kTargetBitrateBps),
2875 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002876
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002877 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002878 AdaptingFrameForwarder source;
2879 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002880 video_stream_encoder_->SetSource(&source,
2881 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002882
Åsa Persson8c1bf952018-09-13 10:42:19 +02002883 int64_t timestamp_ms = kFrameIntervalMs;
2884 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002885 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002886 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002887 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2888 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2889
2890 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002891 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002892 timestamp_ms += kFrameIntervalMs;
2893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2894 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002895 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2896 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2897 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2898
2899 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002900 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002901 timestamp_ms += kFrameIntervalMs;
2902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002903 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002904 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002905 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2907
2908 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002910 timestamp_ms += kFrameIntervalMs;
2911 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2912 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002913 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2914 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2915 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2916
2917 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002918 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002919 timestamp_ms += kFrameIntervalMs;
2920 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002921 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002922 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002923 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2924 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2925
mflodmancc3d4422017-08-03 08:27:51 -07002926 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002927}
2928
Sergey Silkin41c650b2019-10-14 13:12:19 +02002929TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2930 fake_encoder_.SetResolutionBitrateLimits(
2931 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2932
2933 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002934 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2935 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2936 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2937 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002938
2939 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2940 AdaptingFrameForwarder source;
2941 source.set_adaptation_enabled(true);
2942 video_stream_encoder_->SetSource(
2943 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2944
2945 // Insert 720p frame.
2946 int64_t timestamp_ms = kFrameIntervalMs;
2947 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2948 WaitForEncodedFrame(1280, 720);
2949
2950 // Reduce bitrate and trigger adapt down.
2951 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002952 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2953 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2954 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2955 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002956 video_stream_encoder_->TriggerQualityLow();
2957
2958 // Insert 720p frame. It should be downscaled and encoded.
2959 timestamp_ms += kFrameIntervalMs;
2960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2961 WaitForEncodedFrame(960, 540);
2962
2963 // Trigger adapt up. Higher resolution should not be requested duo to lack
2964 // of bitrate.
2965 video_stream_encoder_->TriggerQualityHigh();
2966 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2967
2968 // Increase bitrate.
2969 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002970 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2971 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2972 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2973 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002974
2975 // Trigger adapt up. Higher resolution should be requested.
2976 video_stream_encoder_->TriggerQualityHigh();
2977 VerifyFpsMaxResolutionMax(source.sink_wants());
2978
2979 video_stream_encoder_->Stop();
2980}
2981
2982TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2983 fake_encoder_.SetResolutionBitrateLimits(
2984 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2985
2986 // Set bitrate equal to min bitrate of 540p.
2987 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002988 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2989 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2990 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2991 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002992
2993 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2994 AdaptingFrameForwarder source;
2995 source.set_adaptation_enabled(true);
2996 video_stream_encoder_->SetSource(
2997 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2998
2999 // Insert 720p frame. It should be dropped and lower resolution should be
3000 // requested.
3001 int64_t timestamp_ms = kFrameIntervalMs;
3002 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3003 ExpectDroppedFrame();
3004 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3005
3006 // Insert 720p frame. It should be downscaled and encoded.
3007 timestamp_ms += kFrameIntervalMs;
3008 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3009 WaitForEncodedFrame(960, 540);
3010
3011 video_stream_encoder_->Stop();
3012}
3013
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003014class BalancedDegradationTest : public VideoStreamEncoderTest {
3015 protected:
3016 void SetupTest() {
3017 // Reset encoder for field trials to take effect.
3018 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003019 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003020
3021 // Enable BALANCED preference.
3022 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003023 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3024 }
3025
3026 void OnBitrateUpdated(int bitrate_bps) {
Ying Wang9b881ab2020-02-07 14:29:32 +01003027 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003028 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3029 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003030 }
3031
Åsa Persson45b176f2019-09-30 11:19:05 +02003032 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003033 timestamp_ms_ += kFrameIntervalMs;
3034 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003035 }
3036
3037 void InsertFrameAndWaitForEncoded() {
3038 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003039 sink_.WaitForEncodedFrame(timestamp_ms_);
3040 }
3041
3042 const int kWidth = 640; // pixels:640x360=230400
3043 const int kHeight = 360;
3044 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3045 int64_t timestamp_ms_ = 0;
3046 AdaptingFrameForwarder source_;
3047};
3048
3049TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3050 test::ScopedFieldTrials field_trials(
3051 "WebRTC-Video-BalancedDegradationSettings/"
3052 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3053 SetupTest();
3054
3055 // Force input frame rate.
3056 const int kInputFps = 24;
3057 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3058 stats.input_frame_rate = kInputFps;
3059 stats_proxy_->SetMockStats(stats);
3060
Åsa Persson45b176f2019-09-30 11:19:05 +02003061 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003062 VerifyFpsMaxResolutionMax(source_.sink_wants());
3063
3064 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
3065 // Fps diff (input-requested:0) < threshold, expect AdaptDown to return false.
3066 video_stream_encoder_->TriggerQualityLowExpectFalse();
3067 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3068
3069 video_stream_encoder_->Stop();
3070}
3071
3072TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3073 test::ScopedFieldTrials field_trials(
3074 "WebRTC-Video-BalancedDegradationSettings/"
3075 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3076 SetupTest();
3077
3078 // Force input frame rate.
3079 const int kInputFps = 25;
3080 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3081 stats.input_frame_rate = kInputFps;
3082 stats_proxy_->SetMockStats(stats);
3083
Åsa Persson45b176f2019-09-30 11:19:05 +02003084 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003085 VerifyFpsMaxResolutionMax(source_.sink_wants());
3086
3087 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
3088 // Fps diff (input-requested:1) == threshold, expect AdaptDown to return true.
3089 video_stream_encoder_->TriggerQualityLow();
3090 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3091
3092 video_stream_encoder_->Stop();
3093}
3094
3095TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3096 test::ScopedFieldTrials field_trials(
3097 "WebRTC-Video-BalancedDegradationSettings/"
3098 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3099 SetupTest();
3100
3101 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3102
Åsa Persson45b176f2019-09-30 11:19:05 +02003103 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003104 VerifyFpsMaxResolutionMax(source_.sink_wants());
3105
3106 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3107 video_stream_encoder_->TriggerQualityLow();
3108 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3109
3110 video_stream_encoder_->Stop();
3111}
3112
Åsa Perssonccfb3402019-09-25 15:13:04 +02003113TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003114 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003115 "WebRTC-Video-BalancedDegradationSettings/"
3116 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003117 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003118
Åsa Persson1b247f12019-08-14 17:26:39 +02003119 const int kMinBitrateBps = 425000;
3120 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003121 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003122
Åsa Persson45b176f2019-09-30 11:19:05 +02003123 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003124 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003125 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3126
3127 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3128 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003129 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003130 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003131 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3132
3133 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3134 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003135 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003136 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003137 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3138
Åsa Persson30ab0152019-08-27 12:22:33 +02003139 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3140 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003141 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003142 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3143 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003144 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3145
3146 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003147 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003148 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003149 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003150
Åsa Persson30ab0152019-08-27 12:22:33 +02003151 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003152 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003153 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003154 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003155 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003156 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3157
3158 video_stream_encoder_->Stop();
3159}
3160
Åsa Perssonccfb3402019-09-25 15:13:04 +02003161TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003162 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3163 test::ScopedFieldTrials field_trials(
3164 "WebRTC-Video-BalancedDegradationSettings/"
3165 "pixels:57600|129600|230400,fps:7|24|24/");
3166 SetupTest();
3167 OnBitrateUpdated(kLowTargetBitrateBps);
3168
3169 VerifyNoLimitation(source_.sink_wants());
3170
3171 // Insert frame, expect scaled down:
3172 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3173 InsertFrame();
3174 EXPECT_FALSE(WaitForFrame(1000));
3175 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3176 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3177
3178 // Insert frame, expect scaled down:
3179 // resolution (320x180@24fps).
3180 InsertFrame();
3181 EXPECT_FALSE(WaitForFrame(1000));
3182 EXPECT_LT(source_.sink_wants().max_pixel_count,
3183 source_.last_wants().max_pixel_count);
3184 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3185
3186 // Frame should not be dropped (min pixels per frame reached).
3187 InsertFrameAndWaitForEncoded();
3188
3189 video_stream_encoder_->Stop();
3190}
3191
3192TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003193 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003194 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003195 "WebRTC-Video-BalancedDegradationSettings/"
3196 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003197 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003198
Åsa Persson30ab0152019-08-27 12:22:33 +02003199 const int kResolutionMinBitrateBps = 435000;
3200 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003201 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003202
Åsa Persson45b176f2019-09-30 11:19:05 +02003203 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003204 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003205 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3206
3207 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3208 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003209 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003210 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003211 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3212
3213 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3214 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003215 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003216 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003217 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3218
3219 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3220 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003221 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003222 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003223 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3224
Åsa Persson30ab0152019-08-27 12:22:33 +02003225 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3226 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003227 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003228 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003229 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3230
3231 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3232 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003233 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003234 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3235
3236 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003237 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003238 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003239 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003240 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003241 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3242
3243 video_stream_encoder_->Stop();
3244}
3245
Åsa Perssonccfb3402019-09-25 15:13:04 +02003246TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003247 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003248 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003249 "WebRTC-Video-BalancedDegradationSettings/"
3250 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003251 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003252
Åsa Persson30ab0152019-08-27 12:22:33 +02003253 const int kMinBitrateBps = 425000;
3254 const int kTooLowMinBitrateBps = 424000;
3255 const int kResolutionMinBitrateBps = 435000;
3256 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003257 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003258
Åsa Persson45b176f2019-09-30 11:19:05 +02003259 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003260 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003261 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3262
3263 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3264 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003265 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003266 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003267 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3268
3269 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3270 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003271 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003272 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003273 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3274
3275 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3276 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003277 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003278 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003279 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3280
3281 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3282 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003283 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003284 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3285
3286 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003287 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003288 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003289 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003290 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003291 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3292
3293 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003294 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003295 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003296 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003297 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3298
3299 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003300 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003301 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003302 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003303 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003304 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3305
Åsa Persson1b247f12019-08-14 17:26:39 +02003306 video_stream_encoder_->Stop();
3307}
3308
mflodmancc3d4422017-08-03 08:27:51 -07003309TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003310 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3311 const int kWidth = 1280;
3312 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02003313 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003314 DataRate::BitsPerSec(kTargetBitrateBps),
3315 DataRate::BitsPerSec(kTargetBitrateBps),
3316 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003317
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003318 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003319 AdaptingFrameForwarder source;
3320 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003321 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003322 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003323
Åsa Persson8c1bf952018-09-13 10:42:19 +02003324 int64_t timestamp_ms = kFrameIntervalMs;
3325 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003326 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003327 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003328 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3329 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3330 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3331 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3332
3333 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003334 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003335 timestamp_ms += kFrameIntervalMs;
3336 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3337 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003338 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003339 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3340 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3341 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3342 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3343
3344 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003345 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003346 timestamp_ms += kFrameIntervalMs;
3347 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3348 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003349 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003350 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3351 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3352 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3353 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3354
Jonathan Yubc771b72017-12-08 17:04:29 -08003355 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003356 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003357 timestamp_ms += kFrameIntervalMs;
3358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3359 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003360 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003361 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3362 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003363 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003364 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3365
Jonathan Yubc771b72017-12-08 17:04:29 -08003366 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003367 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003368 timestamp_ms += kFrameIntervalMs;
3369 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3370 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003371 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003372 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003373 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3374 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3375 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3376 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3377
Jonathan Yubc771b72017-12-08 17:04:29 -08003378 // Trigger quality adapt down, expect no change (min resolution reached).
3379 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003380 timestamp_ms += kFrameIntervalMs;
3381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3382 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003383 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3384 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3385 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3386 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3388
3389 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003390 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003391 timestamp_ms += kFrameIntervalMs;
3392 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3393 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003394 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003395 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3396 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3397 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3398 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Trigger cpu adapt up, expect upscaled resolution (640x360).
3401 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003402 timestamp_ms += kFrameIntervalMs;
3403 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3404 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003405 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3406 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3407 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3408 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3409 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3410
3411 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3412 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003413 timestamp_ms += kFrameIntervalMs;
3414 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3415 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003416 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003417 last_wants = source.sink_wants();
3418 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003420 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003421 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3422
3423 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003424 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003425 timestamp_ms += kFrameIntervalMs;
3426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3427 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003428 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003429 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003431 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003432 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3433
3434 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003435 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003436 timestamp_ms += kFrameIntervalMs;
3437 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003438 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003439 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003440 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003441 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3442 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003443 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003444 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003445
mflodmancc3d4422017-08-03 08:27:51 -07003446 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003447}
3448
mflodmancc3d4422017-08-03 08:27:51 -07003449TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003450 const int kWidth = 640;
3451 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003452
Erik Språng4c6ca302019-04-08 15:14:01 +02003453 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003454 DataRate::BitsPerSec(kTargetBitrateBps),
3455 DataRate::BitsPerSec(kTargetBitrateBps),
3456 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003457
perkj803d97f2016-11-01 11:45:46 -07003458 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003459 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003460 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003461 }
3462
mflodmancc3d4422017-08-03 08:27:51 -07003463 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003464 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003465 video_source_.IncomingCapturedFrame(CreateFrame(
3466 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003467 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003468 }
3469
mflodmancc3d4422017-08-03 08:27:51 -07003470 video_stream_encoder_->Stop();
3471 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003472 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003473
Ying Wangef3998f2019-12-09 13:06:53 +01003474 EXPECT_METRIC_EQ(
3475 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3476 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003477 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3478}
3479
mflodmancc3d4422017-08-03 08:27:51 -07003480TEST_F(VideoStreamEncoderTest,
3481 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003482 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003483 DataRate::BitsPerSec(kTargetBitrateBps),
3484 DataRate::BitsPerSec(kTargetBitrateBps),
3485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003486 const int kWidth = 640;
3487 const int kHeight = 360;
3488
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003489 video_stream_encoder_->SetSource(&video_source_,
3490 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003491
3492 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3493 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003494 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003495 }
3496
mflodmancc3d4422017-08-03 08:27:51 -07003497 video_stream_encoder_->Stop();
3498 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003499 stats_proxy_.reset();
3500
3501 EXPECT_EQ(0,
3502 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3503}
3504
mflodmancc3d4422017-08-03 08:27:51 -07003505TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003506 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003507 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003508
3509 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003510 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003511 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003512 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3513 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003514
sprang57c2fff2017-01-16 06:24:02 -08003515 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003516 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003517 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003518 DataRate::BitsPerSec(kLowTargetBitrateBps),
3519 DataRate::BitsPerSec(kLowTargetBitrateBps),
3520 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003521
sprang57c2fff2017-01-16 06:24:02 -08003522 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003523 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3524 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003525 VideoBitrateAllocation bitrate_allocation =
3526 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003527 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003528 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003529 // TODO(srte): The use of millisecs here looks like an error, but the tests
3530 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003531 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003532
3533 // Not called on second frame.
3534 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3535 .Times(0);
3536 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003537 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3538 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003539 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003540
3541 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003542 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3543 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003544 const int64_t start_time_ms = rtc::TimeMillis();
3545 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3546 video_source_.IncomingCapturedFrame(
3547 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3548 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003549 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003550 }
3551
3552 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003553 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003554
mflodmancc3d4422017-08-03 08:27:51 -07003555 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003556}
3557
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003558TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3559 // 2 TLs configured, temporal layers supported by encoder.
3560 const int kNumTemporalLayers = 2;
3561 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3562 fake_encoder_.SetTemporalLayersSupported(0, true);
3563
3564 // Bitrate allocated across temporal layers.
3565 const int kTl0Bps = kTargetBitrateBps *
3566 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003567 kNumTemporalLayers, /*temporal_id*/ 0,
3568 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003569 const int kTl1Bps = kTargetBitrateBps *
3570 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003571 kNumTemporalLayers, /*temporal_id*/ 1,
3572 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003573 VideoBitrateAllocation expected_bitrate;
3574 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3575 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3576
3577 VerifyAllocatedBitrate(expected_bitrate);
3578 video_stream_encoder_->Stop();
3579}
3580
3581TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3582 // 2 TLs configured, temporal layers not supported by encoder.
3583 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3584 fake_encoder_.SetTemporalLayersSupported(0, false);
3585
3586 // Temporal layers not supported by the encoder.
3587 // Total bitrate should be at ti:0.
3588 VideoBitrateAllocation expected_bitrate;
3589 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3590
3591 VerifyAllocatedBitrate(expected_bitrate);
3592 video_stream_encoder_->Stop();
3593}
3594
3595TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3596 // 2 TLs configured, temporal layers only supported for first stream.
3597 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3598 fake_encoder_.SetTemporalLayersSupported(0, true);
3599 fake_encoder_.SetTemporalLayersSupported(1, false);
3600
3601 const int kS0Bps = 150000;
3602 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003603 kS0Bps *
3604 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3605 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003606 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003607 kS0Bps *
3608 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3609 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003610 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3611 // Temporal layers not supported by si:1.
3612 VideoBitrateAllocation expected_bitrate;
3613 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3614 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3615 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3616
3617 VerifyAllocatedBitrate(expected_bitrate);
3618 video_stream_encoder_->Stop();
3619}
3620
Niels Möller7dc26b72017-12-06 10:27:48 +01003621TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3622 const int kFrameWidth = 1280;
3623 const int kFrameHeight = 720;
3624 const int kFramerate = 24;
3625
Erik Språng4c6ca302019-04-08 15:14:01 +02003626 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003627 DataRate::BitsPerSec(kTargetBitrateBps),
3628 DataRate::BitsPerSec(kTargetBitrateBps),
3629 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003630 test::FrameForwarder source;
3631 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003632 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003633
3634 // Insert a single frame, triggering initial configuration.
3635 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3636 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3637
3638 EXPECT_EQ(
3639 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3640 kDefaultFramerate);
3641
3642 // Trigger reconfigure encoder (without resetting the entire instance).
3643 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003644 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003645 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3646 video_encoder_config.number_of_streams = 1;
3647 video_encoder_config.video_stream_factory =
3648 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3649 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003650 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3652
3653 // Detector should be updated with fps limit from codec config.
3654 EXPECT_EQ(
3655 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3656 kFramerate);
3657
3658 // Trigger overuse, max framerate should be reduced.
3659 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3660 stats.input_frame_rate = kFramerate;
3661 stats_proxy_->SetMockStats(stats);
3662 video_stream_encoder_->TriggerCpuOveruse();
3663 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3664 int adapted_framerate =
3665 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3666 EXPECT_LT(adapted_framerate, kFramerate);
3667
3668 // Trigger underuse, max framerate should go back to codec configured fps.
3669 // Set extra low fps, to make sure it's actually reset, not just incremented.
3670 stats = stats_proxy_->GetStats();
3671 stats.input_frame_rate = adapted_framerate / 2;
3672 stats_proxy_->SetMockStats(stats);
3673 video_stream_encoder_->TriggerCpuNormalUsage();
3674 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3675 EXPECT_EQ(
3676 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3677 kFramerate);
3678
3679 video_stream_encoder_->Stop();
3680}
3681
3682TEST_F(VideoStreamEncoderTest,
3683 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3684 const int kFrameWidth = 1280;
3685 const int kFrameHeight = 720;
3686 const int kLowFramerate = 15;
3687 const int kHighFramerate = 25;
3688
Erik Språng4c6ca302019-04-08 15:14:01 +02003689 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003690 DataRate::BitsPerSec(kTargetBitrateBps),
3691 DataRate::BitsPerSec(kTargetBitrateBps),
3692 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003693 test::FrameForwarder source;
3694 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003695 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003696
3697 // Trigger initial configuration.
3698 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003699 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003700 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3701 video_encoder_config.number_of_streams = 1;
3702 video_encoder_config.video_stream_factory =
3703 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3704 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3705 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003706 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003707 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3708
3709 EXPECT_EQ(
3710 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3711 kLowFramerate);
3712
3713 // Trigger overuse, max framerate should be reduced.
3714 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3715 stats.input_frame_rate = kLowFramerate;
3716 stats_proxy_->SetMockStats(stats);
3717 video_stream_encoder_->TriggerCpuOveruse();
3718 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3719 int adapted_framerate =
3720 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3721 EXPECT_LT(adapted_framerate, kLowFramerate);
3722
3723 // Reconfigure the encoder with a new (higher max framerate), max fps should
3724 // still respect the adaptation.
3725 video_encoder_config.video_stream_factory =
3726 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3727 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3728 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003729 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003730 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3731
3732 EXPECT_EQ(
3733 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3734 adapted_framerate);
3735
3736 // Trigger underuse, max framerate should go back to codec configured fps.
3737 stats = stats_proxy_->GetStats();
3738 stats.input_frame_rate = adapted_framerate;
3739 stats_proxy_->SetMockStats(stats);
3740 video_stream_encoder_->TriggerCpuNormalUsage();
3741 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3742 EXPECT_EQ(
3743 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3744 kHighFramerate);
3745
3746 video_stream_encoder_->Stop();
3747}
3748
mflodmancc3d4422017-08-03 08:27:51 -07003749TEST_F(VideoStreamEncoderTest,
3750 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003751 const int kFrameWidth = 1280;
3752 const int kFrameHeight = 720;
3753 const int kFramerate = 24;
3754
Erik Språng4c6ca302019-04-08 15:14:01 +02003755 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003756 DataRate::BitsPerSec(kTargetBitrateBps),
3757 DataRate::BitsPerSec(kTargetBitrateBps),
3758 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003759 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003760 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003761 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003762
3763 // Trigger initial configuration.
3764 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003765 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003766 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3767 video_encoder_config.number_of_streams = 1;
3768 video_encoder_config.video_stream_factory =
3769 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3770 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003771 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003772 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003773 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003774
Niels Möller7dc26b72017-12-06 10:27:48 +01003775 EXPECT_EQ(
3776 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3777 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003778
3779 // Trigger overuse, max framerate should be reduced.
3780 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3781 stats.input_frame_rate = kFramerate;
3782 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003783 video_stream_encoder_->TriggerCpuOveruse();
3784 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003785 int adapted_framerate =
3786 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003787 EXPECT_LT(adapted_framerate, kFramerate);
3788
3789 // Change degradation preference to not enable framerate scaling. Target
3790 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003791 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003792 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003793 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003794 EXPECT_EQ(
3795 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3796 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003797
mflodmancc3d4422017-08-03 08:27:51 -07003798 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003799}
3800
mflodmancc3d4422017-08-03 08:27:51 -07003801TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003802 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003803 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003804 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3805 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3806 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003807 const int kWidth = 640;
3808 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003809
asaperssonfab67072017-04-04 05:51:49 -07003810 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003811
3812 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003813 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003814
3815 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003816 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003817
sprangc5d62e22017-04-02 23:53:04 -07003818 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003819
asaperssonfab67072017-04-04 05:51:49 -07003820 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003821 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003822 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003823
3824 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003825 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003826
sprangc5d62e22017-04-02 23:53:04 -07003827 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003828
mflodmancc3d4422017-08-03 08:27:51 -07003829 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003830}
3831
mflodmancc3d4422017-08-03 08:27:51 -07003832TEST_F(VideoStreamEncoderTest,
3833 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003834 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003835 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003836 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3837 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3838 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003839 const int kWidth = 640;
3840 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003841
3842 // We expect the n initial frames to get dropped.
3843 int i;
3844 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003845 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003846 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003847 }
3848 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003849 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003850 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003851
3852 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003853 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003854
mflodmancc3d4422017-08-03 08:27:51 -07003855 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003856}
3857
mflodmancc3d4422017-08-03 08:27:51 -07003858TEST_F(VideoStreamEncoderTest,
3859 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003860 const int kWidth = 640;
3861 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003862 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003863 DataRate::BitsPerSec(kLowTargetBitrateBps),
3864 DataRate::BitsPerSec(kLowTargetBitrateBps),
3865 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003866
3867 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003868 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003869 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003870
asaperssonfab67072017-04-04 05:51:49 -07003871 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003872 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003873 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003874
mflodmancc3d4422017-08-03 08:27:51 -07003875 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003876}
3877
mflodmancc3d4422017-08-03 08:27:51 -07003878TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003879 const int kWidth = 640;
3880 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003881 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003882
3883 VideoEncoderConfig video_encoder_config;
3884 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3885 // Make format different, to force recreation of encoder.
3886 video_encoder_config.video_format.parameters["foo"] = "foo";
3887 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003888 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003889 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003890 DataRate::BitsPerSec(kLowTargetBitrateBps),
3891 DataRate::BitsPerSec(kLowTargetBitrateBps),
3892 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003893
kthelgasonb83797b2017-02-14 11:57:25 -08003894 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003895 video_stream_encoder_->SetSource(&video_source_,
3896 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003897
asaperssonfab67072017-04-04 05:51:49 -07003898 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003899 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003900 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003901
mflodmancc3d4422017-08-03 08:27:51 -07003902 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003903 fake_encoder_.SetQualityScaling(true);
3904}
3905
Åsa Persson139f4dc2019-08-02 09:29:58 +02003906TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3907 webrtc::test::ScopedFieldTrials field_trials(
3908 "WebRTC-Video-QualityScalerSettings/"
3909 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3910 // Reset encoder for field trials to take effect.
3911 ConfigureEncoder(video_encoder_config_.Copy());
3912 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3913 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3914 const int kWidth = 640;
3915 const int kHeight = 360;
3916
3917 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003918 DataRate::BitsPerSec(kTargetBitrateBps),
3919 DataRate::BitsPerSec(kTargetBitrateBps),
3920 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003921 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3922 // Frame should not be dropped.
3923 WaitForEncodedFrame(1);
3924
3925 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003926 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3927 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3928 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003929 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3930 // Frame should not be dropped.
3931 WaitForEncodedFrame(2);
3932
3933 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003934 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3935 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3936 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003937 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3938 // Expect to drop this frame, the wait should time out.
3939 ExpectDroppedFrame();
3940
3941 // Expect the sink_wants to specify a scaled frame.
3942 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3943 video_stream_encoder_->Stop();
3944}
3945
Åsa Perssone644a032019-11-08 15:56:00 +01003946TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3947 webrtc::test::ScopedFieldTrials field_trials(
3948 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3949
3950 // Reset encoder for field trials to take effect.
3951 VideoEncoderConfig config = video_encoder_config_.Copy();
3952 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02003953 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01003954 ConfigureEncoder(std::move(config));
3955 fake_encoder_.SetQp(kQpLow);
3956
3957 // Enable MAINTAIN_FRAMERATE preference.
3958 AdaptingFrameForwarder source;
3959 source.set_adaptation_enabled(true);
3960 video_stream_encoder_->SetSource(&source,
3961 DegradationPreference::MAINTAIN_FRAMERATE);
3962
3963 // Start at low bitrate.
3964 const int kLowBitrateBps = 200000;
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003965 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(kLowBitrateBps),
3966 DataRate::BitsPerSec(kLowBitrateBps),
3967 DataRate::BitsPerSec(kLowBitrateBps),
3968 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003969
3970 // Expect first frame to be dropped and resolution to be limited.
3971 const int kWidth = 1280;
3972 const int kHeight = 720;
3973 const int64_t kFrameIntervalMs = 100;
3974 int64_t timestamp_ms = kFrameIntervalMs;
3975 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3976 ExpectDroppedFrame();
3977 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3978
3979 // Increase bitrate to encoder max.
Evan Shrubsoledff79252020-04-16 11:34:32 +02003980 video_stream_encoder_->OnBitrateUpdated(max_bitrate, max_bitrate, max_bitrate,
3981 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003982
3983 // Insert frames and advance |min_duration_ms|.
3984 for (size_t i = 1; i <= 10; i++) {
3985 timestamp_ms += kFrameIntervalMs;
3986 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3987 WaitForEncodedFrame(timestamp_ms);
3988 }
3989 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3990 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3991
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003992 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01003993
3994 // Insert frame should trigger high BW and release quality limitation.
3995 timestamp_ms += kFrameIntervalMs;
3996 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3997 WaitForEncodedFrame(timestamp_ms);
3998 VerifyFpsMaxResolutionMax(source.sink_wants());
3999
4000 // Frame should not be adapted.
4001 timestamp_ms += kFrameIntervalMs;
4002 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4003 WaitForEncodedFrame(kWidth, kHeight);
4004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4005
4006 video_stream_encoder_->Stop();
4007}
4008
mflodmancc3d4422017-08-03 08:27:51 -07004009TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004010 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4011 const int kTooSmallWidth = 10;
4012 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02004013 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004014 DataRate::BitsPerSec(kTargetBitrateBps),
4015 DataRate::BitsPerSec(kTargetBitrateBps),
4016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004017
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004018 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004019 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004020 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004021 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004022 VerifyNoLimitation(source.sink_wants());
4023 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4024
4025 // Trigger adapt down, too small frame, expect no change.
4026 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004027 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004028 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004029 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07004030 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4031 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4032
mflodmancc3d4422017-08-03 08:27:51 -07004033 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004034}
4035
mflodmancc3d4422017-08-03 08:27:51 -07004036TEST_F(VideoStreamEncoderTest,
4037 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004038 const int kTooSmallWidth = 10;
4039 const int kTooSmallHeight = 10;
4040 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02004041 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004042 DataRate::BitsPerSec(kTargetBitrateBps),
4043 DataRate::BitsPerSec(kTargetBitrateBps),
4044 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004045
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004046 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004047 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004048 video_stream_encoder_->SetSource(&source,
4049 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004050 VerifyNoLimitation(source.sink_wants());
4051 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4052 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4053
4054 // Trigger adapt down, expect limited framerate.
4055 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004056 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004057 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004058 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4059 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4060 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4061 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4062
4063 // Trigger adapt down, too small frame, expect no change.
4064 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004065 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004066 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004067 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4068 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4069 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4070 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4071
mflodmancc3d4422017-08-03 08:27:51 -07004072 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004073}
4074
mflodmancc3d4422017-08-03 08:27:51 -07004075TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004076 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02004077 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004078 DataRate::BitsPerSec(kTargetBitrateBps),
4079 DataRate::BitsPerSec(kTargetBitrateBps),
4080 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004081 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004082 const int kFrameWidth = 1280;
4083 const int kFrameHeight = 720;
4084 video_source_.IncomingCapturedFrame(
4085 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004086 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004087 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004088}
4089
sprangb1ca0732017-02-01 08:38:12 -08004090// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004091TEST_F(VideoStreamEncoderTest,
4092 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004093 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004094 DataRate::BitsPerSec(kTargetBitrateBps),
4095 DataRate::BitsPerSec(kTargetBitrateBps),
4096 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004097
4098 const int kFrameWidth = 1280;
4099 const int kFrameHeight = 720;
4100 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004101 // requested by
4102 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004103 video_source_.set_adaptation_enabled(true);
4104
4105 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004106 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004107 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004108
4109 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004110 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004111 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004112 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004113 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004114
asaperssonfab67072017-04-04 05:51:49 -07004115 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004116 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08004117 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004118 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004119 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004120
mflodmancc3d4422017-08-03 08:27:51 -07004121 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004122}
sprangfe627f32017-03-29 08:24:59 -07004123
mflodmancc3d4422017-08-03 08:27:51 -07004124TEST_F(VideoStreamEncoderTest,
4125 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004126 const int kFrameWidth = 1280;
4127 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004128
Erik Språng4c6ca302019-04-08 15:14:01 +02004129 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004130 DataRate::BitsPerSec(kTargetBitrateBps),
4131 DataRate::BitsPerSec(kTargetBitrateBps),
4132 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004133 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004134 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004135 video_source_.set_adaptation_enabled(true);
4136
sprang4847ae62017-06-27 07:06:52 -07004137 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004138
4139 video_source_.IncomingCapturedFrame(
4140 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004141 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004142
4143 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004144 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004145
4146 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004147 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004148 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004149 video_source_.IncomingCapturedFrame(
4150 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004151 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004152 }
4153
4154 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004155 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004156 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004157 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004158 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004159 video_source_.IncomingCapturedFrame(
4160 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004161 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004162 ++num_frames_dropped;
4163 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004164 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004165 }
4166 }
4167
sprang4847ae62017-06-27 07:06:52 -07004168 // Add some slack to account for frames dropped by the frame dropper.
4169 const int kErrorMargin = 1;
4170 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004171 kErrorMargin);
4172
4173 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004174 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004175 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004176 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004177 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004178 video_source_.IncomingCapturedFrame(
4179 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004180 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004181 ++num_frames_dropped;
4182 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004183 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004184 }
4185 }
sprang4847ae62017-06-27 07:06:52 -07004186 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004187 kErrorMargin);
4188
4189 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07004190 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07004191 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004192 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004193 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004194 video_source_.IncomingCapturedFrame(
4195 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004196 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004197 ++num_frames_dropped;
4198 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004199 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004200 }
4201 }
sprang4847ae62017-06-27 07:06:52 -07004202 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004203 kErrorMargin);
4204
4205 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07004206 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07004207 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004208 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004209 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004210 video_source_.IncomingCapturedFrame(
4211 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004212 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004213 ++num_frames_dropped;
4214 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004215 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004216 }
4217 }
4218 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4219
mflodmancc3d4422017-08-03 08:27:51 -07004220 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004221}
4222
mflodmancc3d4422017-08-03 08:27:51 -07004223TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004224 const int kFramerateFps = 5;
4225 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004226 const int kFrameWidth = 1280;
4227 const int kFrameHeight = 720;
4228
sprang4847ae62017-06-27 07:06:52 -07004229 // Reconfigure encoder with two temporal layers and screensharing, which will
4230 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004231 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004232
Erik Språng4c6ca302019-04-08 15:14:01 +02004233 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004234 DataRate::BitsPerSec(kTargetBitrateBps),
4235 DataRate::BitsPerSec(kTargetBitrateBps),
4236 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004237 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004238 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004239 video_source_.set_adaptation_enabled(true);
4240
sprang4847ae62017-06-27 07:06:52 -07004241 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004242
4243 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004244 rtc::VideoSinkWants last_wants;
4245 do {
4246 last_wants = video_source_.sink_wants();
4247
sprangc5d62e22017-04-02 23:53:04 -07004248 // Insert frames to get a new fps estimate...
4249 for (int j = 0; j < kFramerateFps; ++j) {
4250 video_source_.IncomingCapturedFrame(
4251 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004252 if (video_source_.last_sent_width()) {
4253 sink_.WaitForEncodedFrame(timestamp_ms);
4254 }
sprangc5d62e22017-04-02 23:53:04 -07004255 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004256 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004257 }
4258 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004259 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004260 } while (video_source_.sink_wants().max_framerate_fps <
4261 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004262
Jonathan Yubc771b72017-12-08 17:04:29 -08004263 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004264
mflodmancc3d4422017-08-03 08:27:51 -07004265 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004266}
asaperssonf7e294d2017-06-13 23:25:22 -07004267
mflodmancc3d4422017-08-03 08:27:51 -07004268TEST_F(VideoStreamEncoderTest,
4269 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004270 const int kWidth = 1280;
4271 const int kHeight = 720;
4272 const int64_t kFrameIntervalMs = 150;
4273 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004274 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004275 DataRate::BitsPerSec(kTargetBitrateBps),
4276 DataRate::BitsPerSec(kTargetBitrateBps),
4277 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004278
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004279 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004280 AdaptingFrameForwarder source;
4281 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004282 video_stream_encoder_->SetSource(&source,
4283 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004284 timestamp_ms += kFrameIntervalMs;
4285 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004286 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004287 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004288 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4289 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4290 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4291
4292 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004293 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004294 timestamp_ms += kFrameIntervalMs;
4295 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004296 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004297 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4298 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4300 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4301
4302 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004303 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004304 timestamp_ms += kFrameIntervalMs;
4305 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004306 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004307 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4308 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4309 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4310 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4311
4312 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004313 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004314 timestamp_ms += kFrameIntervalMs;
4315 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004316 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004317 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4318 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4319 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4320 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4321
4322 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004323 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004324 timestamp_ms += kFrameIntervalMs;
4325 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004326 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004327 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4329 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4330 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4331
4332 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004333 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004334 timestamp_ms += kFrameIntervalMs;
4335 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004336 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004337 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4339 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4340 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4341
4342 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004343 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004344 timestamp_ms += kFrameIntervalMs;
4345 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004346 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004347 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4349 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4350 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4351
4352 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004353 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004354 timestamp_ms += kFrameIntervalMs;
4355 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004356 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004357 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4358 rtc::VideoSinkWants last_wants = source.sink_wants();
4359 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4360 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4361 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4362
4363 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004364 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004365 timestamp_ms += kFrameIntervalMs;
4366 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004367 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004368 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4369 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4370 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4371 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4372
4373 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004374 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004375 timestamp_ms += kFrameIntervalMs;
4376 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004377 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004378 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4380 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4381 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4382
4383 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004384 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004385 timestamp_ms += kFrameIntervalMs;
4386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004387 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004388 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4390 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4391 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4392
4393 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004394 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004395 timestamp_ms += kFrameIntervalMs;
4396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004397 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004398 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4399 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4401 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4402
4403 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004404 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004405 timestamp_ms += kFrameIntervalMs;
4406 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004407 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004408 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4410 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4411 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4412
4413 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004414 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004415 timestamp_ms += kFrameIntervalMs;
4416 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004417 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004418 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4420 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4421 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4422
4423 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004424 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004425 timestamp_ms += kFrameIntervalMs;
4426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004427 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004428 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4429 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4430 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4431 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4432
Åsa Persson30ab0152019-08-27 12:22:33 +02004433 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004434 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004435 timestamp_ms += kFrameIntervalMs;
4436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004437 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004438 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004439 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004440 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4441 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4442 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4443
4444 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004445 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004446 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004447 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4448
mflodmancc3d4422017-08-03 08:27:51 -07004449 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004450}
4451
mflodmancc3d4422017-08-03 08:27:51 -07004452TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004453 const int kWidth = 1280;
4454 const int kHeight = 720;
4455 const int64_t kFrameIntervalMs = 150;
4456 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004457 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004458 DataRate::BitsPerSec(kTargetBitrateBps),
4459 DataRate::BitsPerSec(kTargetBitrateBps),
4460 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004461
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004462 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004463 AdaptingFrameForwarder source;
4464 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004465 video_stream_encoder_->SetSource(&source,
4466 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004467 timestamp_ms += kFrameIntervalMs;
4468 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004469 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004470 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004471 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4472 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4473 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4474 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4475 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4476 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4477
4478 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004479 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004480 timestamp_ms += kFrameIntervalMs;
4481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004482 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004483 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4484 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4486 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4487 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4488 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4489 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4490
4491 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004492 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004493 timestamp_ms += kFrameIntervalMs;
4494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004495 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004496 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4499 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4500 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4501 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4502 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4503
4504 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004505 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004506 timestamp_ms += kFrameIntervalMs;
4507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004508 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004509 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4510 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4512 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4513 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4514 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4515 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4516
4517 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004518 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004519 timestamp_ms += kFrameIntervalMs;
4520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004521 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004522 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4525 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4526 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4527 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4528 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4529
4530 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004531 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004532 timestamp_ms += kFrameIntervalMs;
4533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004534 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004535 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4538 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4539 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4540 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4541 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4542
4543 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004544 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004545 timestamp_ms += kFrameIntervalMs;
4546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004547 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004548 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004549 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4553 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4554 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4555 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4556
4557 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004558 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004559 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004560 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4561 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4562
mflodmancc3d4422017-08-03 08:27:51 -07004563 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004564}
4565
mflodmancc3d4422017-08-03 08:27:51 -07004566TEST_F(VideoStreamEncoderTest,
4567 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004568 const int kWidth = 640;
4569 const int kHeight = 360;
4570 const int kFpsLimit = 15;
4571 const int64_t kFrameIntervalMs = 150;
4572 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004573 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004574 DataRate::BitsPerSec(kTargetBitrateBps),
4575 DataRate::BitsPerSec(kTargetBitrateBps),
4576 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004577
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004578 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004579 AdaptingFrameForwarder source;
4580 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004581 video_stream_encoder_->SetSource(&source,
4582 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004583 timestamp_ms += kFrameIntervalMs;
4584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004585 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004586 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004587 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4588 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4589 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4590 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4591 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4592 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4593
4594 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004595 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004596 timestamp_ms += kFrameIntervalMs;
4597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004598 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004599 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4600 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4601 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4602 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4603 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4604 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4605 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4606
4607 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004608 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004609 timestamp_ms += kFrameIntervalMs;
4610 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004611 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004612 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4614 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4617 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4618 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4619
4620 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004621 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004622 timestamp_ms += kFrameIntervalMs;
4623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004624 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004625 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4626 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4630 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4631 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4632
4633 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004634 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004635 timestamp_ms += kFrameIntervalMs;
4636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004637 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004638 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004639 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4640 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4641 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4642 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4643 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4644 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4645
4646 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004647 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004648 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004649 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4650 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4651
mflodmancc3d4422017-08-03 08:27:51 -07004652 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004653}
4654
mflodmancc3d4422017-08-03 08:27:51 -07004655TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004656 const int kFrameWidth = 1920;
4657 const int kFrameHeight = 1080;
4658 // 3/4 of 1920.
4659 const int kAdaptedFrameWidth = 1440;
4660 // 3/4 of 1080 rounded down to multiple of 4.
4661 const int kAdaptedFrameHeight = 808;
4662 const int kFramerate = 24;
4663
Erik Språng4c6ca302019-04-08 15:14:01 +02004664 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004665 DataRate::BitsPerSec(kTargetBitrateBps),
4666 DataRate::BitsPerSec(kTargetBitrateBps),
4667 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004668 // Trigger reconfigure encoder (without resetting the entire instance).
4669 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004670 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004671 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4672 video_encoder_config.number_of_streams = 1;
4673 video_encoder_config.video_stream_factory =
4674 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004675 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004676 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004677 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004678
4679 video_source_.set_adaptation_enabled(true);
4680
4681 video_source_.IncomingCapturedFrame(
4682 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004683 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004684
4685 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004686 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004687 video_source_.IncomingCapturedFrame(
4688 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004689 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004690
mflodmancc3d4422017-08-03 08:27:51 -07004691 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004692}
4693
mflodmancc3d4422017-08-03 08:27:51 -07004694TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004695 const int kFrameWidth = 1280;
4696 const int kFrameHeight = 720;
4697 const int kLowFps = 2;
4698 const int kHighFps = 30;
4699
Erik Språng4c6ca302019-04-08 15:14:01 +02004700 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004701 DataRate::BitsPerSec(kTargetBitrateBps),
4702 DataRate::BitsPerSec(kTargetBitrateBps),
4703 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004704
4705 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4706 max_framerate_ = kLowFps;
4707
4708 // Insert 2 seconds of 2fps video.
4709 for (int i = 0; i < kLowFps * 2; ++i) {
4710 video_source_.IncomingCapturedFrame(
4711 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4712 WaitForEncodedFrame(timestamp_ms);
4713 timestamp_ms += 1000 / kLowFps;
4714 }
4715
4716 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004717 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004718 DataRate::BitsPerSec(kTargetBitrateBps),
4719 DataRate::BitsPerSec(kTargetBitrateBps),
4720 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004721 video_source_.IncomingCapturedFrame(
4722 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4723 WaitForEncodedFrame(timestamp_ms);
4724 timestamp_ms += 1000 / kLowFps;
4725
4726 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4727
4728 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004729 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004730 const int kFrameIntervalMs = 1000 / kHighFps;
4731 max_framerate_ = kHighFps;
4732 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4733 video_source_.IncomingCapturedFrame(
4734 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4735 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4736 // be dropped if the encoder hans't been updated with the new higher target
4737 // framerate yet, causing it to overshoot the target bitrate and then
4738 // suffering the wrath of the media optimizer.
4739 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4740 timestamp_ms += kFrameIntervalMs;
4741 }
4742
4743 // Don expect correct measurement just yet, but it should be higher than
4744 // before.
4745 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4746
mflodmancc3d4422017-08-03 08:27:51 -07004747 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004748}
4749
mflodmancc3d4422017-08-03 08:27:51 -07004750TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004751 const int kFrameWidth = 1280;
4752 const int kFrameHeight = 720;
4753 const int kTargetBitrateBps = 1000000;
4754
4755 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004756 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004757 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004758 DataRate::BitsPerSec(kTargetBitrateBps),
4759 DataRate::BitsPerSec(kTargetBitrateBps),
4760 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004761 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004762
4763 // Insert a first video frame, causes another bitrate update.
4764 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4765 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4766 video_source_.IncomingCapturedFrame(
4767 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4768 WaitForEncodedFrame(timestamp_ms);
4769
4770 // Next, simulate video suspension due to pacer queue overrun.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004771 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
4772 DataRate::BitsPerSec(0),
4773 DataRate::BitsPerSec(0), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004774
4775 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004776 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004777 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004778
4779 // Bitrate observer should not be called.
4780 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4781 video_source_.IncomingCapturedFrame(
4782 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4783 ExpectDroppedFrame();
4784
mflodmancc3d4422017-08-03 08:27:51 -07004785 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004786}
ilnik6b826ef2017-06-16 06:53:48 -07004787
Niels Möller4db138e2018-04-19 09:04:13 +02004788TEST_F(VideoStreamEncoderTest,
4789 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4790 const int kFrameWidth = 1280;
4791 const int kFrameHeight = 720;
4792 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004793 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004794 DataRate::BitsPerSec(kTargetBitrateBps),
4795 DataRate::BitsPerSec(kTargetBitrateBps),
4796 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004797 video_source_.IncomingCapturedFrame(
4798 CreateFrame(1, kFrameWidth, kFrameHeight));
4799 WaitForEncodedFrame(1);
4800 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4801 .low_encode_usage_threshold_percent,
4802 default_options.low_encode_usage_threshold_percent);
4803 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4804 .high_encode_usage_threshold_percent,
4805 default_options.high_encode_usage_threshold_percent);
4806 video_stream_encoder_->Stop();
4807}
4808
4809TEST_F(VideoStreamEncoderTest,
4810 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4811 const int kFrameWidth = 1280;
4812 const int kFrameHeight = 720;
4813 CpuOveruseOptions hardware_options;
4814 hardware_options.low_encode_usage_threshold_percent = 150;
4815 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004816 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004817
Erik Språng4c6ca302019-04-08 15:14:01 +02004818 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004819 DataRate::BitsPerSec(kTargetBitrateBps),
4820 DataRate::BitsPerSec(kTargetBitrateBps),
4821 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004822 video_source_.IncomingCapturedFrame(
4823 CreateFrame(1, kFrameWidth, kFrameHeight));
4824 WaitForEncodedFrame(1);
4825 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4826 .low_encode_usage_threshold_percent,
4827 hardware_options.low_encode_usage_threshold_percent);
4828 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4829 .high_encode_usage_threshold_percent,
4830 hardware_options.high_encode_usage_threshold_percent);
4831 video_stream_encoder_->Stop();
4832}
4833
Niels Möller6bb5ab92019-01-11 11:11:10 +01004834TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4835 const int kFrameWidth = 320;
4836 const int kFrameHeight = 240;
4837 const int kFps = 30;
4838 const int kTargetBitrateBps = 120000;
4839 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4840
Erik Språng4c6ca302019-04-08 15:14:01 +02004841 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004842 DataRate::BitsPerSec(kTargetBitrateBps),
4843 DataRate::BitsPerSec(kTargetBitrateBps),
4844 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004845
4846 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4847 max_framerate_ = kFps;
4848
4849 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4850 fake_encoder_.SimulateOvershoot(1.0);
4851 int num_dropped = 0;
4852 for (int i = 0; i < kNumFramesInRun; ++i) {
4853 video_source_.IncomingCapturedFrame(
4854 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4855 // Wait up to two frame durations for a frame to arrive.
4856 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4857 ++num_dropped;
4858 }
4859 timestamp_ms += 1000 / kFps;
4860 }
4861
Erik Språnga8d48ab2019-02-08 14:17:40 +01004862 // Framerate should be measured to be near the expected target rate.
4863 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4864
4865 // Frame drops should be within 5% of expected 0%.
4866 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004867
4868 // Make encoder produce frames at double the expected bitrate during 3 seconds
4869 // of video, verify number of drops. Rate needs to be slightly changed in
4870 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004871 double overshoot_factor = 2.0;
4872 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4873 // With bitrate adjuster, when need to overshoot even more to trigger
4874 // frame dropping.
4875 overshoot_factor *= 2;
4876 }
4877 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004878 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004879 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4880 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4881 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004882 num_dropped = 0;
4883 for (int i = 0; i < kNumFramesInRun; ++i) {
4884 video_source_.IncomingCapturedFrame(
4885 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4886 // Wait up to two frame durations for a frame to arrive.
4887 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4888 ++num_dropped;
4889 }
4890 timestamp_ms += 1000 / kFps;
4891 }
4892
Erik Språng4c6ca302019-04-08 15:14:01 +02004893 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004894 DataRate::BitsPerSec(kTargetBitrateBps),
4895 DataRate::BitsPerSec(kTargetBitrateBps),
4896 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004897
4898 // Target framerate should be still be near the expected target, despite
4899 // the frame drops.
4900 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4901
4902 // Frame drops should be within 5% of expected 50%.
4903 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004904
4905 video_stream_encoder_->Stop();
4906}
4907
4908TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4909 const int kFrameWidth = 320;
4910 const int kFrameHeight = 240;
4911 const int kActualInputFps = 24;
4912 const int kTargetBitrateBps = 120000;
4913
4914 ASSERT_GT(max_framerate_, kActualInputFps);
4915
4916 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4917 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004918 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004919 DataRate::BitsPerSec(kTargetBitrateBps),
4920 DataRate::BitsPerSec(kTargetBitrateBps),
4921 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004922
4923 // Insert 3 seconds of video, with an input fps lower than configured max.
4924 for (int i = 0; i < kActualInputFps * 3; ++i) {
4925 video_source_.IncomingCapturedFrame(
4926 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4927 // Wait up to two frame durations for a frame to arrive.
4928 WaitForEncodedFrame(timestamp_ms);
4929 timestamp_ms += 1000 / kActualInputFps;
4930 }
4931
4932 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4933
4934 video_stream_encoder_->Stop();
4935}
4936
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004937TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4938 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004939 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004940 DataRate::BitsPerSec(kTargetBitrateBps),
4941 DataRate::BitsPerSec(kTargetBitrateBps),
4942 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004943
4944 fake_encoder_.BlockNextEncode();
4945 video_source_.IncomingCapturedFrame(
4946 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4947 WaitForEncodedFrame(1);
4948 // On the very first frame full update should be forced.
4949 rect = fake_encoder_.GetLastUpdateRect();
4950 EXPECT_EQ(rect.offset_x, 0);
4951 EXPECT_EQ(rect.offset_y, 0);
4952 EXPECT_EQ(rect.height, codec_height_);
4953 EXPECT_EQ(rect.width, codec_width_);
4954 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4955 // call to ContinueEncode.
4956 video_source_.IncomingCapturedFrame(
4957 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4958 ExpectDroppedFrame();
4959 video_source_.IncomingCapturedFrame(
4960 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4961 ExpectDroppedFrame();
4962 fake_encoder_.ContinueEncode();
4963 WaitForEncodedFrame(3);
4964 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4965 rect = fake_encoder_.GetLastUpdateRect();
4966 EXPECT_EQ(rect.offset_x, 1);
4967 EXPECT_EQ(rect.offset_y, 0);
4968 EXPECT_EQ(rect.width, 10);
4969 EXPECT_EQ(rect.height, 1);
4970
4971 video_source_.IncomingCapturedFrame(
4972 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4973 WaitForEncodedFrame(4);
4974 // Previous frame was encoded, so no accumulation should happen.
4975 rect = fake_encoder_.GetLastUpdateRect();
4976 EXPECT_EQ(rect.offset_x, 0);
4977 EXPECT_EQ(rect.offset_y, 0);
4978 EXPECT_EQ(rect.width, 1);
4979 EXPECT_EQ(rect.height, 1);
4980
4981 video_stream_encoder_->Stop();
4982}
4983
Erik Språngd7329ca2019-02-21 21:19:53 +01004984TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004985 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004986 DataRate::BitsPerSec(kTargetBitrateBps),
4987 DataRate::BitsPerSec(kTargetBitrateBps),
4988 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004989
4990 // First frame is always keyframe.
4991 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4992 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004993 EXPECT_THAT(
4994 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004995 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004996
4997 // Insert delta frame.
4998 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4999 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005000 EXPECT_THAT(
5001 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005002 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005003
5004 // Request next frame be a key-frame.
5005 video_stream_encoder_->SendKeyFrame();
5006 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5007 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005008 EXPECT_THAT(
5009 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005010 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005011
5012 video_stream_encoder_->Stop();
5013}
5014
5015TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5016 // Setup simulcast with three streams.
5017 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01005018 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005019 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5020 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5021 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005022 // Wait for all three layers before triggering event.
5023 sink_.SetNumExpectedLayers(3);
5024
5025 // First frame is always keyframe.
5026 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5027 WaitForEncodedFrame(1);
5028 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005029 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5030 VideoFrameType::kVideoFrameKey,
5031 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005032
5033 // Insert delta frame.
5034 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5035 WaitForEncodedFrame(2);
5036 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005037 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5038 VideoFrameType::kVideoFrameDelta,
5039 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005040
5041 // Request next frame be a key-frame.
5042 // Only first stream is configured to produce key-frame.
5043 video_stream_encoder_->SendKeyFrame();
5044 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5045 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005046
5047 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5048 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005049 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005050 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005051 VideoFrameType::kVideoFrameKey,
5052 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005053
5054 video_stream_encoder_->Stop();
5055}
5056
5057TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5058 // Configure internal source factory and setup test again.
5059 encoder_factory_.SetHasInternalSource(true);
5060 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005061 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005062 DataRate::BitsPerSec(kTargetBitrateBps),
5063 DataRate::BitsPerSec(kTargetBitrateBps),
5064 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005065
5066 // Call encoder directly, simulating internal source where encoded frame
5067 // callback in VideoStreamEncoder is called despite no OnFrame().
5068 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5069 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005070 EXPECT_THAT(
5071 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005072 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005073
Niels Möller8f7ce222019-03-21 15:43:58 +01005074 const std::vector<VideoFrameType> kDeltaFrame = {
5075 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005076 // Need to set timestamp manually since manually for injected frame.
5077 VideoFrame frame = CreateFrame(101, nullptr);
5078 frame.set_timestamp(101);
5079 fake_encoder_.InjectFrame(frame, false);
5080 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005081 EXPECT_THAT(
5082 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005083 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005084
5085 // Request key-frame. The forces a dummy frame down into the encoder.
5086 fake_encoder_.ExpectNullFrame();
5087 video_stream_encoder_->SendKeyFrame();
5088 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005089 EXPECT_THAT(
5090 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005091 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005092
5093 video_stream_encoder_->Stop();
5094}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005095
5096TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5097 // Configure internal source factory and setup test again.
5098 encoder_factory_.SetHasInternalSource(true);
5099 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005100 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005101 DataRate::BitsPerSec(kTargetBitrateBps),
5102 DataRate::BitsPerSec(kTargetBitrateBps),
5103 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005104
5105 int64_t timestamp = 1;
5106 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005107 image.SetEncodedData(
5108 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005109 image.capture_time_ms_ = ++timestamp;
5110 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5111 const int64_t kEncodeFinishDelayMs = 10;
5112 image.timing_.encode_start_ms = timestamp;
5113 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5114 fake_encoder_.InjectEncodedImage(image);
5115 // Wait for frame without incrementing clock.
5116 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5117 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5118 // capture timestamp should be kEncodeFinishDelayMs in the past.
5119 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5120 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5121 kEncodeFinishDelayMs);
5122
5123 video_stream_encoder_->Stop();
5124}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005125
5126TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5127 // Configure internal source factory and setup test again.
5128 encoder_factory_.SetHasInternalSource(true);
5129 ResetEncoder("H264", 1, 1, 1, false);
5130
5131 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5132 image._frameType = VideoFrameType::kVideoFrameKey;
5133
5134 CodecSpecificInfo codec_specific_info;
5135 codec_specific_info.codecType = kVideoCodecH264;
5136
5137 RTPFragmentationHeader fragmentation;
5138 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5139 fragmentation.fragmentationOffset[0] = 4;
5140 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5141
5142 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5143 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5144
5145 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5146 testing::ElementsAreArray(optimal_sps));
5147 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5148 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5149 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5150 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5151
5152 video_stream_encoder_->Stop();
5153}
5154
5155TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5156 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5157 0x00, 0x00, 0x03, 0x03, 0xF4,
5158 0x05, 0x03, 0xC7, 0xC0};
5159
5160 // Configure internal source factory and setup test again.
5161 encoder_factory_.SetHasInternalSource(true);
5162 ResetEncoder("H264", 1, 1, 1, false);
5163
5164 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5165 image._frameType = VideoFrameType::kVideoFrameKey;
5166
5167 CodecSpecificInfo codec_specific_info;
5168 codec_specific_info.codecType = kVideoCodecH264;
5169
5170 RTPFragmentationHeader fragmentation;
5171 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5172 fragmentation.fragmentationOffset[0] = 4;
5173 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5174
5175 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5176 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5177
5178 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5179 testing::ElementsAreArray(optimal_sps));
5180 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5181 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5182 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5183 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5184
5185 video_stream_encoder_->Stop();
5186}
5187
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005188TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5189 const int kFrameWidth = 1280;
5190 const int kFrameHeight = 720;
5191 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5192
5193 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005194 DataRate::BitsPerSec(kTargetBitrateBps),
5195 DataRate::BitsPerSec(kTargetBitrateBps),
5196 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005197 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5198
5199 // Insert a first video frame. It should be dropped because of downscale in
5200 // resolution.
5201 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5202 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5203 frame.set_rotation(kVideoRotation_270);
5204 video_source_.IncomingCapturedFrame(frame);
5205
5206 ExpectDroppedFrame();
5207
5208 // Second frame is downscaled.
5209 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5210 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5211 frame.set_rotation(kVideoRotation_90);
5212 video_source_.IncomingCapturedFrame(frame);
5213
5214 WaitForEncodedFrame(timestamp_ms);
5215 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5216
5217 // Insert another frame, also downscaled.
5218 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5219 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5220 frame.set_rotation(kVideoRotation_180);
5221 video_source_.IncomingCapturedFrame(frame);
5222
5223 WaitForEncodedFrame(timestamp_ms);
5224 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5225
5226 video_stream_encoder_->Stop();
5227}
5228
Erik Språng5056af02019-09-02 15:53:11 +02005229TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5230 const int kFrameWidth = 320;
5231 const int kFrameHeight = 180;
5232
5233 // Initial rate.
5234 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005235 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5236 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5237 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005238 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005239 /*rtt_ms=*/0,
5240 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005241
5242 // Insert a first video frame so that encoder gets configured.
5243 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5244 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5245 frame.set_rotation(kVideoRotation_270);
5246 video_source_.IncomingCapturedFrame(frame);
5247 WaitForEncodedFrame(timestamp_ms);
5248
5249 // Set a target rate below the minimum allowed by the codec settings.
5250 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005251 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5252 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Erik Språng5056af02019-09-02 15:53:11 +02005253 video_stream_encoder_->OnBitrateUpdated(
5254 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005255 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005256 /*link_allocation=*/target_rate,
5257 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005258 /*rtt_ms=*/0,
5259 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005260 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5261
5262 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5263 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5264 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005265 DataRate allocation_sum =
5266 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005267 EXPECT_EQ(min_rate, allocation_sum);
5268 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5269
5270 video_stream_encoder_->Stop();
5271}
5272
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005273TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005274 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005275 DataRate::BitsPerSec(kTargetBitrateBps),
5276 DataRate::BitsPerSec(kTargetBitrateBps),
5277 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005278 // Capture a frame and wait for it to synchronize with the encoder thread.
5279 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5280 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5281 WaitForEncodedFrame(1);
5282
5283 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5284 ASSERT_TRUE(prev_rate_settings.has_value());
5285 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5286 kDefaultFramerate);
5287
5288 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5289 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5290 timestamp_ms += 1000 / kDefaultFramerate;
5291 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5292 WaitForEncodedFrame(timestamp_ms);
5293 }
5294 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5295 kDefaultFramerate);
5296 // Capture larger frame to trigger a reconfigure.
5297 codec_height_ *= 2;
5298 codec_width_ *= 2;
5299 timestamp_ms += 1000 / kDefaultFramerate;
5300 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5301 WaitForEncodedFrame(timestamp_ms);
5302
5303 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5304 auto current_rate_settings =
5305 fake_encoder_.GetAndResetLastRateControlSettings();
5306 // Ensure we have actually reconfigured twice
5307 // The rate settings should have been set again even though
5308 // they haven't changed.
5309 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005310 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005311
5312 video_stream_encoder_->Stop();
5313}
5314
philipeld9cc8c02019-09-16 14:53:40 +02005315struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5316 MOCK_METHOD0(RequestEncoderFallback, void());
5317 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 11:30:00 +01005318 MOCK_METHOD1(RequestEncoderSwitch,
5319 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 14:53:40 +02005320};
5321
5322TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5323 constexpr int kDontCare = 100;
5324
5325 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5326 video_send_config_.encoder_settings.encoder_switch_request_callback =
5327 &switch_callback;
5328 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5329 encoder_config.codec_type = kVideoCodecVP8;
5330 webrtc::test::ScopedFieldTrials field_trial(
5331 "WebRTC-NetworkCondition-EncoderSwitch/"
5332 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5333 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5334
5335 // Reset encoder for new configuration to take effect.
5336 ConfigureEncoder(std::move(encoder_config));
5337
5338 // Send one frame to trigger ReconfigureEncoder.
5339 video_source_.IncomingCapturedFrame(
5340 CreateFrame(kDontCare, kDontCare, kDontCare));
5341
5342 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005343 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5344 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005345 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005346 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005347
5348 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005349 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5350 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5351 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005352 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005353 /*rtt_ms=*/0,
5354 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005355
5356 video_stream_encoder_->Stop();
5357}
5358
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005359TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5360 constexpr int kDontCare = 100;
5361
5362 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5363 video_send_config_.encoder_settings.encoder_switch_request_callback =
5364 &switch_callback;
5365 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5366 encoder_config.codec_type = kVideoCodecVP8;
5367 webrtc::test::ScopedFieldTrials field_trial(
5368 "WebRTC-NetworkCondition-EncoderSwitch/"
5369 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5370 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5371
5372 // Reset encoder for new configuration to take effect.
5373 ConfigureEncoder(std::move(encoder_config));
5374
5375 // Send one frame to trigger ReconfigureEncoder.
5376 video_source_.IncomingCapturedFrame(
5377 CreateFrame(kDontCare, kDontCare, kDontCare));
5378
5379 using Config = EncoderSwitchRequestCallback::Config;
5380 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5381 .Times(0);
5382
5383 video_stream_encoder_->OnBitrateUpdated(
5384 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5385 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5386 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5387 /*fraction_lost=*/0,
5388 /*rtt_ms=*/0,
5389 /*cwnd_reduce_ratio=*/0);
5390
5391 video_stream_encoder_->Stop();
5392}
5393
philipeld9cc8c02019-09-16 14:53:40 +02005394TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5395 constexpr int kSufficientBitrateToNotDrop = 1000;
5396 constexpr int kHighRes = 500;
5397 constexpr int kLowRes = 100;
5398
5399 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5400 video_send_config_.encoder_settings.encoder_switch_request_callback =
5401 &switch_callback;
5402 webrtc::test::ScopedFieldTrials field_trial(
5403 "WebRTC-NetworkCondition-EncoderSwitch/"
5404 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5405 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5406 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5407 encoder_config.codec_type = kVideoCodecH264;
5408
5409 // Reset encoder for new configuration to take effect.
5410 ConfigureEncoder(std::move(encoder_config));
5411
5412 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5413 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5414 // not fail.
5415 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005416 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5417 /*stable_target_bitrate=*/
5418 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5419 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005420 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005421 /*rtt_ms=*/0,
5422 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005423
5424 // Send one frame to trigger ReconfigureEncoder.
5425 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5426 WaitForEncodedFrame(1);
5427
5428 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005429 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5430 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005431 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005432 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005433
5434 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5435 WaitForEncodedFrame(2);
5436
5437 video_stream_encoder_->Stop();
5438}
5439
philipel9b058032020-02-10 11:30:00 +01005440TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5441 constexpr int kDontCare = 100;
5442 StrictMock<MockEncoderSelector> encoder_selector;
5443 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5444 &fake_encoder_, &encoder_selector);
5445 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5446
5447 // Reset encoder for new configuration to take effect.
5448 ConfigureEncoder(video_encoder_config_.Copy());
5449
5450 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5451
5452 video_source_.IncomingCapturedFrame(
5453 CreateFrame(kDontCare, kDontCare, kDontCare));
5454 video_stream_encoder_->Stop();
5455
5456 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5457 // to it's factory, so in order for the encoder instance in the
5458 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5459 // reset the |video_stream_encoder_| here.
5460 video_stream_encoder_.reset();
5461}
5462
5463TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5464 constexpr int kDontCare = 100;
5465
5466 NiceMock<MockEncoderSelector> encoder_selector;
5467 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5468 video_send_config_.encoder_settings.encoder_switch_request_callback =
5469 &switch_callback;
5470 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5471 &fake_encoder_, &encoder_selector);
5472 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5473
5474 // Reset encoder for new configuration to take effect.
5475 ConfigureEncoder(video_encoder_config_.Copy());
5476
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005477 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005478 .WillByDefault(Return(SdpVideoFormat("AV1")));
5479 EXPECT_CALL(switch_callback,
5480 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5481 Field(&SdpVideoFormat::name, "AV1"))));
5482
5483 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005484 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5485 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5486 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005487 /*fraction_lost=*/0,
5488 /*rtt_ms=*/0,
5489 /*cwnd_reduce_ratio=*/0);
5490
5491 video_stream_encoder_->Stop();
5492}
5493
5494TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5495 constexpr int kSufficientBitrateToNotDrop = 1000;
5496 constexpr int kDontCare = 100;
5497
5498 NiceMock<MockVideoEncoder> video_encoder;
5499 NiceMock<MockEncoderSelector> encoder_selector;
5500 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5501 video_send_config_.encoder_settings.encoder_switch_request_callback =
5502 &switch_callback;
5503 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5504 &video_encoder, &encoder_selector);
5505 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5506
5507 // Reset encoder for new configuration to take effect.
5508 ConfigureEncoder(video_encoder_config_.Copy());
5509
5510 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5511 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5512 // not fail.
5513 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005514 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5515 /*stable_target_bitrate=*/
5516 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5517 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005518 /*fraction_lost=*/0,
5519 /*rtt_ms=*/0,
5520 /*cwnd_reduce_ratio=*/0);
5521
5522 ON_CALL(video_encoder, Encode(_, _))
5523 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5524 ON_CALL(encoder_selector, OnEncoderBroken())
5525 .WillByDefault(Return(SdpVideoFormat("AV2")));
5526
5527 rtc::Event encode_attempted;
5528 EXPECT_CALL(switch_callback,
5529 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5530 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5531 EXPECT_EQ(format.name, "AV2");
5532 encode_attempted.Set();
5533 });
5534
5535 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5536 encode_attempted.Wait(3000);
5537
5538 video_stream_encoder_->Stop();
5539
5540 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5541 // to it's factory, so in order for the encoder instance in the
5542 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5543 // reset the |video_stream_encoder_| here.
5544 video_stream_encoder_.reset();
5545}
5546
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005547TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005548 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005549 const int kFrameWidth = 320;
5550 const int kFrameHeight = 180;
5551
5552 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005553 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005554 video_stream_encoder_->OnBitrateUpdated(
5555 /*target_bitrate=*/rate,
5556 /*stable_target_bitrate=*/rate,
5557 /*link_allocation=*/rate,
5558 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005559 /*rtt_ms=*/0,
5560 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005561
5562 // Insert a first video frame so that encoder gets configured.
5563 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5564 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5565 frame.set_rotation(kVideoRotation_270);
5566 video_source_.IncomingCapturedFrame(frame);
5567 WaitForEncodedFrame(timestamp_ms);
5568 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5569
5570 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005571 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005572 video_stream_encoder_->OnBitrateUpdated(
5573 /*target_bitrate=*/new_stable_rate,
5574 /*stable_target_bitrate=*/new_stable_rate,
5575 /*link_allocation=*/rate,
5576 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005577 /*rtt_ms=*/0,
5578 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005579 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5580 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5581 video_stream_encoder_->Stop();
5582}
5583
5584TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005585 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005586 const int kFrameWidth = 320;
5587 const int kFrameHeight = 180;
5588
5589 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005590 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005591 video_stream_encoder_->OnBitrateUpdated(
5592 /*target_bitrate=*/rate,
5593 /*stable_target_bitrate=*/rate,
5594 /*link_allocation=*/rate,
5595 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005596 /*rtt_ms=*/0,
5597 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005598
5599 // Insert a first video frame so that encoder gets configured.
5600 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5601 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5602 frame.set_rotation(kVideoRotation_270);
5603 video_source_.IncomingCapturedFrame(frame);
5604 WaitForEncodedFrame(timestamp_ms);
5605 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5606
5607 // Set a higher target rate without changing the link_allocation. Should not
5608 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005609 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005610 video_stream_encoder_->OnBitrateUpdated(
5611 /*target_bitrate=*/rate,
5612 /*stable_target_bitrate=*/new_stable_rate,
5613 /*link_allocation=*/rate,
5614 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005615 /*rtt_ms=*/0,
5616 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005617 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5618 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5619 video_stream_encoder_->Stop();
5620}
5621
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005622TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5623 test::ScopedFieldTrials field_trials(
5624 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5625 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5626 const int kFramerateFps = 30;
5627 const int kWidth = 1920;
5628 const int kHeight = 1080;
5629 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5630 // Works on screenshare mode.
5631 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5632 // We rely on the automatic resolution adaptation, but we handle framerate
5633 // adaptation manually by mocking the stats proxy.
5634 video_source_.set_adaptation_enabled(true);
5635
5636 // BALANCED degradation preference is required for this feature.
5637 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005638 DataRate::BitsPerSec(kTargetBitrateBps),
5639 DataRate::BitsPerSec(kTargetBitrateBps),
5640 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005641 video_stream_encoder_->SetSource(&video_source_,
5642 webrtc::DegradationPreference::BALANCED);
5643 VerifyNoLimitation(video_source_.sink_wants());
5644
5645 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5646 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5647
5648 // Pass enough frames with the full update to trigger animation detection.
5649 for (int i = 0; i < kNumFrames; ++i) {
5650 int64_t timestamp_ms =
5651 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5652 frame.set_ntp_time_ms(timestamp_ms);
5653 frame.set_timestamp_us(timestamp_ms * 1000);
5654 video_source_.IncomingCapturedFrame(frame);
5655 WaitForEncodedFrame(timestamp_ms);
5656 }
5657
5658 // Resolution should be limited.
5659 rtc::VideoSinkWants expected;
5660 expected.max_framerate_fps = kFramerateFps;
5661 expected.max_pixel_count = 1280 * 720 + 1;
5662 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5663
5664 // Pass one frame with no known update.
5665 // Resolution cap should be removed immediately.
5666 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5667 frame.set_ntp_time_ms(timestamp_ms);
5668 frame.set_timestamp_us(timestamp_ms * 1000);
5669 frame.clear_update_rect();
5670
5671 video_source_.IncomingCapturedFrame(frame);
5672 WaitForEncodedFrame(timestamp_ms);
5673
5674 // Resolution should be unlimited now.
5675 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5676
5677 video_stream_encoder_->Stop();
5678}
5679
perkj26091b12016-09-01 01:17:40 -07005680} // namespace webrtc