blob: e48ccd806a051528a372aff2c7f11f667ddd328a [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Erik Språngf93eda12019-01-16 17:10:57 +010024#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020025#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020026#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010027#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010028#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020029#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070030#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080031#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020032#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010033#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "rtc_base/fake_clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020035#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010037#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020038#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "system_wrappers/include/sleep.h"
40#include "test/encoder_settings.h"
41#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020042#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010043#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020044#include "test/gmock.h"
45#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020046#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020047#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070048
49namespace webrtc {
50
sprangb1ca0732017-02-01 08:38:12 -080051using ScaleReason = AdaptationObserverInterface::AdaptReason;
sprang57c2fff2017-01-16 06:24:02 -080052using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020053using ::testing::AllOf;
54using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010055using ::testing::Matcher;
56using ::testing::NiceMock;
57using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020058using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080059
perkj803d97f2016-11-01 11:45:46 -070060namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020061const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010062const int kQpLow = 1;
63const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020064const int kMinFramerateFps = 2;
65const int kMinBalancedFramerateFps = 7;
66const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080067const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010068const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020069const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010070const uint32_t kSimulcastTargetBitrateBps = 3150000;
71const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080072const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070073const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020075const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020076const VideoEncoder::ResolutionBitrateLimits
77 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
78const VideoEncoder::ResolutionBitrateLimits
79 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080080
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020081uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
82 0x00, 0x00, 0x03, 0x03, 0xF4,
83 0x05, 0x03, 0xC7, 0xE0, 0x1B,
84 0x41, 0x10, 0x8D, 0x00};
85
perkj803d97f2016-11-01 11:45:46 -070086class TestBuffer : public webrtc::I420Buffer {
87 public:
88 TestBuffer(rtc::Event* event, int width, int height)
89 : I420Buffer(width, height), event_(event) {}
90
91 private:
92 friend class rtc::RefCountedObject<TestBuffer>;
93 ~TestBuffer() override {
94 if (event_)
95 event_->Set();
96 }
97 rtc::Event* const event_;
98};
99
Noah Richards51db4212019-06-12 06:59:12 -0700100// A fake native buffer that can't be converted to I420.
101class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
102 public:
103 FakeNativeBuffer(rtc::Event* event, int width, int height)
104 : event_(event), width_(width), height_(height) {}
105 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
106 int width() const override { return width_; }
107 int height() const override { return height_; }
108 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
109 return nullptr;
110 }
111
112 private:
113 friend class rtc::RefCountedObject<FakeNativeBuffer>;
114 ~FakeNativeBuffer() override {
115 if (event_)
116 event_->Set();
117 }
118 rtc::Event* const event_;
119 const int width_;
120 const int height_;
121};
122
Niels Möller7dc26b72017-12-06 10:27:48 +0100123class CpuOveruseDetectorProxy : public OveruseFrameDetector {
124 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200125 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
126 : OveruseFrameDetector(metrics_observer),
Niels Möller7dc26b72017-12-06 10:27:48 +0100127 last_target_framerate_fps_(-1) {}
128 virtual ~CpuOveruseDetectorProxy() {}
129
130 void OnTargetFramerateUpdated(int framerate_fps) override {
131 rtc::CritScope cs(&lock_);
132 last_target_framerate_fps_ = framerate_fps;
133 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
134 }
135
136 int GetLastTargetFramerate() {
137 rtc::CritScope cs(&lock_);
138 return last_target_framerate_fps_;
139 }
140
Niels Möller4db138e2018-04-19 09:04:13 +0200141 CpuOveruseOptions GetOptions() { return options_; }
142
Niels Möller7dc26b72017-12-06 10:27:48 +0100143 private:
144 rtc::CriticalSection lock_;
145 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
146};
147
mflodmancc3d4422017-08-03 08:27:51 -0700148class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700149 public:
Niels Möller213618e2018-07-24 09:29:58 +0200150 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200151 const VideoStreamEncoderSettings& settings,
152 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100153 : VideoStreamEncoder(Clock::GetRealTimeClock(),
154 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200155 stats_proxy,
156 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200157 std::unique_ptr<OveruseFrameDetector>(
158 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100159 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100160 task_queue_factory),
161 fake_cpu_resource_(
162 std::make_unique<FakeResource>(ResourceUsageState::kStable,
163 "FakeResource[CPU]")),
164 fake_quality_resource_(
165 std::make_unique<FakeResource>(ResourceUsageState::kStable,
166 "FakeResource[QP]")) {
167 InjectAdaptationResource(
168 fake_quality_resource_.get(),
169 AdaptationObserverInterface::AdaptReason::kQuality);
170 InjectAdaptationResource(fake_cpu_resource_.get(),
171 AdaptationObserverInterface::AdaptReason::kCpu);
172 }
perkj803d97f2016-11-01 11:45:46 -0700173
Henrik Boströmb08882b2020-01-07 10:11:17 +0100174 void PostTaskAndWait(bool down,
175 AdaptationObserverInterface::AdaptReason reason) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200176 PostTaskAndWait(down, reason, /*expected_results=*/true);
177 }
178
Henrik Boströmb08882b2020-01-07 10:11:17 +0100179 void PostTaskAndWait(bool down,
180 AdaptationObserverInterface::AdaptReason reason,
181 bool expected_results) {
Niels Möllerc572ff32018-11-07 08:43:50 +0100182 rtc::Event event;
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200183 encoder_queue()->PostTask([this, &event, reason, down, expected_results] {
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100184 ResourceUsageState usage_state =
185 down ? ResourceUsageState::kOveruse : ResourceUsageState::kUnderuse;
186
187 FakeResource* resource = nullptr;
188 switch (reason) {
189 case AdaptationObserverInterface::kQuality:
190 resource = fake_quality_resource_.get();
191 break;
192 case AdaptationObserverInterface::kCpu:
193 resource = fake_cpu_resource_.get();
194 break;
195 default:
196 RTC_NOTREACHED();
197 }
198
199 resource->set_usage_state(usage_state);
200 if (!expected_results) {
201 ASSERT_EQ(AdaptationObserverInterface::kQuality, reason)
202 << "We can only assert adaptation result for quality resources";
203 EXPECT_EQ(
204 ResourceListenerResponse::kQualityScalerShouldIncreaseFrequency,
205 resource->last_response());
206 } else {
207 EXPECT_EQ(ResourceListenerResponse::kNothing,
208 resource->last_response());
209 }
210
perkj803d97f2016-11-01 11:45:46 -0700211 event.Set();
212 });
perkj070ba852017-02-16 15:46:27 -0800213 ASSERT_TRUE(event.Wait(5000));
perkj803d97f2016-11-01 11:45:46 -0700214 }
215
kthelgason2fc52542017-03-03 00:24:41 -0800216 // This is used as a synchronisation mechanism, to make sure that the
217 // encoder queue is not blocked before we start sending it frames.
218 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100219 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200220 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800221 ASSERT_TRUE(event.Wait(5000));
222 }
223
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200224 void TriggerCpuOveruse() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100225 PostTaskAndWait(/*down=*/true,
226 AdaptationObserverInterface::AdaptReason::kCpu);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200227 }
kthelgason876222f2016-11-29 01:44:11 -0800228
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200229 void TriggerCpuNormalUsage() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100230 PostTaskAndWait(/*down=*/false,
231 AdaptationObserverInterface::AdaptReason::kCpu);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200232 }
kthelgason876222f2016-11-29 01:44:11 -0800233
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200234 void TriggerQualityLow() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100235 PostTaskAndWait(/*down=*/true,
236 AdaptationObserverInterface::AdaptReason::kQuality);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200237 }
kthelgason876222f2016-11-29 01:44:11 -0800238
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200239 void TriggerQualityLowExpectFalse() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100240 PostTaskAndWait(/*down=*/true,
241 AdaptationObserverInterface::AdaptReason::kQuality,
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200242 /*expected_results=*/false);
243 }
244
245 void TriggerQualityHigh() {
Henrik Boströmb08882b2020-01-07 10:11:17 +0100246 PostTaskAndWait(/*down=*/false,
247 AdaptationObserverInterface::AdaptReason::kQuality);
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200248 }
sprangfda496a2017-06-15 04:21:07 -0700249
Niels Möller7dc26b72017-12-06 10:27:48 +0100250 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100251 std::unique_ptr<FakeResource> fake_cpu_resource_;
252 std::unique_ptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700253};
254
asapersson5f7226f2016-11-25 04:37:00 -0800255class VideoStreamFactory
256 : public VideoEncoderConfig::VideoStreamFactoryInterface {
257 public:
sprangfda496a2017-06-15 04:21:07 -0700258 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
259 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800260 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700261 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800262 }
263
264 private:
265 std::vector<VideoStream> CreateEncoderStreams(
266 int width,
267 int height,
268 const VideoEncoderConfig& encoder_config) override {
269 std::vector<VideoStream> streams =
270 test::CreateVideoStreams(width, height, encoder_config);
271 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100272 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700273 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800274 }
275 return streams;
276 }
sprangfda496a2017-06-15 04:21:07 -0700277
asapersson5f7226f2016-11-25 04:37:00 -0800278 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700279 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800280};
281
Noah Richards51db4212019-06-12 06:59:12 -0700282// Simulates simulcast behavior and makes highest stream resolutions divisible
283// by 4.
284class CroppingVideoStreamFactory
285 : public VideoEncoderConfig::VideoStreamFactoryInterface {
286 public:
287 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
288 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
289 EXPECT_GT(num_temporal_layers, 0u);
290 EXPECT_GT(framerate, 0);
291 }
292
293 private:
294 std::vector<VideoStream> CreateEncoderStreams(
295 int width,
296 int height,
297 const VideoEncoderConfig& encoder_config) override {
298 std::vector<VideoStream> streams = test::CreateVideoStreams(
299 width - width % 4, height - height % 4, encoder_config);
300 for (VideoStream& stream : streams) {
301 stream.num_temporal_layers = num_temporal_layers_;
302 stream.max_framerate = framerate_;
303 }
304 return streams;
305 }
306
307 const size_t num_temporal_layers_;
308 const int framerate_;
309};
310
sprangb1ca0732017-02-01 08:38:12 -0800311class AdaptingFrameForwarder : public test::FrameForwarder {
312 public:
313 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700314 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800315
316 void set_adaptation_enabled(bool enabled) {
317 rtc::CritScope cs(&crit_);
318 adaptation_enabled_ = enabled;
319 }
320
asaperssonfab67072017-04-04 05:51:49 -0700321 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800322 rtc::CritScope cs(&crit_);
323 return adaptation_enabled_;
324 }
325
asapersson09f05612017-05-15 23:40:18 -0700326 rtc::VideoSinkWants last_wants() const {
327 rtc::CritScope cs(&crit_);
328 return last_wants_;
329 }
330
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200331 absl::optional<int> last_sent_width() const { return last_width_; }
332 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800333
sprangb1ca0732017-02-01 08:38:12 -0800334 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
335 int cropped_width = 0;
336 int cropped_height = 0;
337 int out_width = 0;
338 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700339 if (adaption_enabled()) {
340 if (adapter_.AdaptFrameResolution(
341 video_frame.width(), video_frame.height(),
342 video_frame.timestamp_us() * 1000, &cropped_width,
343 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100344 VideoFrame adapted_frame =
345 VideoFrame::Builder()
346 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
347 nullptr, out_width, out_height))
348 .set_timestamp_rtp(99)
349 .set_timestamp_ms(99)
350 .set_rotation(kVideoRotation_0)
351 .build();
sprangc5d62e22017-04-02 23:53:04 -0700352 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100353 if (video_frame.has_update_rect()) {
354 adapted_frame.set_update_rect(
355 video_frame.update_rect().ScaleWithFrame(
356 video_frame.width(), video_frame.height(), 0, 0,
357 video_frame.width(), video_frame.height(), out_width,
358 out_height));
359 }
sprangc5d62e22017-04-02 23:53:04 -0700360 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800361 last_width_.emplace(adapted_frame.width());
362 last_height_.emplace(adapted_frame.height());
363 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200364 last_width_ = absl::nullopt;
365 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700366 }
sprangb1ca0732017-02-01 08:38:12 -0800367 } else {
368 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800369 last_width_.emplace(video_frame.width());
370 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800371 }
372 }
373
374 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
375 const rtc::VideoSinkWants& wants) override {
376 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700377 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100378 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800379 test::FrameForwarder::AddOrUpdateSink(sink, wants);
380 }
sprangb1ca0732017-02-01 08:38:12 -0800381 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700382 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
383 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200384 absl::optional<int> last_width_;
385 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800386};
sprangc5d62e22017-04-02 23:53:04 -0700387
Niels Möller213618e2018-07-24 09:29:58 +0200388// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700389class MockableSendStatisticsProxy : public SendStatisticsProxy {
390 public:
391 MockableSendStatisticsProxy(Clock* clock,
392 const VideoSendStream::Config& config,
393 VideoEncoderConfig::ContentType content_type)
394 : SendStatisticsProxy(clock, config, content_type) {}
395
396 VideoSendStream::Stats GetStats() override {
397 rtc::CritScope cs(&lock_);
398 if (mock_stats_)
399 return *mock_stats_;
400 return SendStatisticsProxy::GetStats();
401 }
402
Niels Möller213618e2018-07-24 09:29:58 +0200403 int GetInputFrameRate() const override {
404 rtc::CritScope cs(&lock_);
405 if (mock_stats_)
406 return mock_stats_->input_frame_rate;
407 return SendStatisticsProxy::GetInputFrameRate();
408 }
sprangc5d62e22017-04-02 23:53:04 -0700409 void SetMockStats(const VideoSendStream::Stats& stats) {
410 rtc::CritScope cs(&lock_);
411 mock_stats_.emplace(stats);
412 }
413
414 void ResetMockStats() {
415 rtc::CritScope cs(&lock_);
416 mock_stats_.reset();
417 }
418
419 private:
420 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200421 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700422};
423
sprang4847ae62017-06-27 07:06:52 -0700424class MockBitrateObserver : public VideoBitrateAllocationObserver {
425 public:
Erik Språng566124a2018-04-23 12:32:22 +0200426 MOCK_METHOD1(OnBitrateAllocationUpdated, void(const VideoBitrateAllocation&));
sprang4847ae62017-06-27 07:06:52 -0700427};
428
philipel9b058032020-02-10 11:30:00 +0100429class MockEncoderSelector
430 : public VideoEncoderFactory::EncoderSelectorInterface {
431 public:
432 MOCK_METHOD1(OnCurrentEncoder, void(const SdpVideoFormat& format));
Mirta Dvornicic4f34d782020-02-26 13:01:19 +0100433 MOCK_METHOD1(OnAvailableBitrate,
philipel9b058032020-02-10 11:30:00 +0100434 absl::optional<SdpVideoFormat>(const DataRate& rate));
435 MOCK_METHOD0(OnEncoderBroken, absl::optional<SdpVideoFormat>());
436};
437
perkj803d97f2016-11-01 11:45:46 -0700438} // namespace
439
mflodmancc3d4422017-08-03 08:27:51 -0700440class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700441 public:
442 static const int kDefaultTimeoutMs = 30 * 1000;
443
mflodmancc3d4422017-08-03 08:27:51 -0700444 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700445 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700446 codec_width_(320),
447 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200448 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200449 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700450 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200451 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700452 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700453 Clock::GetRealTimeClock(),
454 video_send_config_,
455 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700456 sink_(&fake_encoder_) {}
457
458 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700459 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700460 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200461 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800462 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200463 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200464 video_send_config_.rtp.payload_name = "FAKE";
465 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700466
Per512ecb32016-09-23 15:52:06 +0200467 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200468 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700469 video_encoder_config.video_stream_factory =
470 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100471 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700472
473 // Framerate limit is specified by the VideoStreamFactory.
474 std::vector<VideoStream> streams =
475 video_encoder_config.video_stream_factory->CreateEncoderStreams(
476 codec_width_, codec_height_, video_encoder_config);
477 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100478 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700479
Niels Möllerf1338562018-04-26 09:51:47 +0200480 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800481 }
482
Niels Möllerf1338562018-04-26 09:51:47 +0200483 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700484 if (video_stream_encoder_)
485 video_stream_encoder_->Stop();
486 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200487 stats_proxy_.get(), video_send_config_.encoder_settings,
488 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700489 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
490 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700491 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700492 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
493 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200494 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700495 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800496 }
497
498 void ResetEncoder(const std::string& payload_name,
499 size_t num_streams,
500 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700501 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700502 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200503 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800504
505 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200506 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800507 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100508 video_encoder_config.max_bitrate_bps =
509 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800510 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700511 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
512 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700513 video_encoder_config.content_type =
514 screenshare ? VideoEncoderConfig::ContentType::kScreen
515 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700516 if (payload_name == "VP9") {
517 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
518 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
519 video_encoder_config.encoder_specific_settings =
520 new rtc::RefCountedObject<
521 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
522 }
Niels Möllerf1338562018-04-26 09:51:47 +0200523 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700524 }
525
sprang57c2fff2017-01-16 06:24:02 -0800526 VideoFrame CreateFrame(int64_t ntp_time_ms,
527 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100528 VideoFrame frame =
529 VideoFrame::Builder()
530 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
531 destruction_event, codec_width_, codec_height_))
532 .set_timestamp_rtp(99)
533 .set_timestamp_ms(99)
534 .set_rotation(kVideoRotation_0)
535 .build();
sprang57c2fff2017-01-16 06:24:02 -0800536 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700537 return frame;
538 }
539
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100540 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
541 rtc::Event* destruction_event,
542 int offset_x) const {
543 VideoFrame frame =
544 VideoFrame::Builder()
545 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
546 destruction_event, codec_width_, codec_height_))
547 .set_timestamp_rtp(99)
548 .set_timestamp_ms(99)
549 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100550 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100551 .build();
552 frame.set_ntp_time_ms(ntp_time_ms);
553 return frame;
554 }
555
sprang57c2fff2017-01-16 06:24:02 -0800556 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100557 VideoFrame frame =
558 VideoFrame::Builder()
559 .set_video_frame_buffer(
560 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
561 .set_timestamp_rtp(99)
562 .set_timestamp_ms(99)
563 .set_rotation(kVideoRotation_0)
564 .build();
sprang57c2fff2017-01-16 06:24:02 -0800565 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700566 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700567 return frame;
568 }
569
Noah Richards51db4212019-06-12 06:59:12 -0700570 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
571 rtc::Event* destruction_event,
572 int width,
573 int height) const {
574 VideoFrame frame =
575 VideoFrame::Builder()
576 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
577 destruction_event, width, height))
578 .set_timestamp_rtp(99)
579 .set_timestamp_ms(99)
580 .set_rotation(kVideoRotation_0)
581 .build();
582 frame.set_ntp_time_ms(ntp_time_ms);
583 return frame;
584 }
585
586 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
587 rtc::Event* destruction_event) const {
588 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
589 codec_height_);
590 }
591
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100592 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
593 MockBitrateObserver bitrate_observer;
594 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
595
596 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
597 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +0200598 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100599 DataRate::BitsPerSec(kTargetBitrateBps),
600 DataRate::BitsPerSec(kTargetBitrateBps),
601 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100602
603 video_source_.IncomingCapturedFrame(
604 CreateFrame(1, codec_width_, codec_height_));
605 WaitForEncodedFrame(1);
606 }
607
asapersson02465b82017-04-10 01:12:52 -0700608 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700609 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700610 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
611 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700612 }
613
asapersson09f05612017-05-15 23:40:18 -0700614 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
615 const rtc::VideoSinkWants& wants2) {
616 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
617 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
618 }
619
Åsa Persson8c1bf952018-09-13 10:42:19 +0200620 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
621 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
622 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
623 EXPECT_FALSE(wants.target_pixel_count);
624 }
625
asapersson09f05612017-05-15 23:40:18 -0700626 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
627 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200628 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700629 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
630 EXPECT_GT(wants1.max_pixel_count, 0);
631 }
632
633 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
634 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200635 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700636 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
637 }
638
asaperssonf7e294d2017-06-13 23:25:22 -0700639 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
640 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200641 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700642 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
643 }
644
645 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
646 const rtc::VideoSinkWants& wants2) {
647 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
648 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
649 }
650
651 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
652 const rtc::VideoSinkWants& wants2) {
653 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
654 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
655 }
656
657 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
658 const rtc::VideoSinkWants& wants2) {
659 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
660 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
661 EXPECT_GT(wants1.max_pixel_count, 0);
662 }
663
664 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
665 const rtc::VideoSinkWants& wants2) {
666 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
667 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
668 }
669
asapersson09f05612017-05-15 23:40:18 -0700670 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
671 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200672 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700673 EXPECT_LT(wants.max_pixel_count, pixel_count);
674 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700675 }
676
677 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
678 EXPECT_LT(wants.max_framerate_fps, fps);
679 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
680 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700681 }
682
asaperssonf7e294d2017-06-13 23:25:22 -0700683 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
684 int expected_fps) {
685 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
686 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
687 EXPECT_FALSE(wants.target_pixel_count);
688 }
689
Jonathan Yubc771b72017-12-08 17:04:29 -0800690 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
691 int last_frame_pixels) {
692 // Balanced mode should always scale FPS to the desired range before
693 // attempting to scale resolution.
694 int fps_limit = wants.max_framerate_fps;
695 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100696 EXPECT_LE(7, fps_limit);
697 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800698 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100699 EXPECT_LE(10, fps_limit);
700 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800701 } else if (last_frame_pixels <= 640 * 480) {
702 EXPECT_LE(15, fps_limit);
703 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200704 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800705 }
706 }
707
sprang4847ae62017-06-27 07:06:52 -0700708 void WaitForEncodedFrame(int64_t expected_ntp_time) {
709 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100710 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700711 }
712
713 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
714 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100715 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700716 return ok;
717 }
718
719 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
720 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100721 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700722 }
723
724 void ExpectDroppedFrame() {
725 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100726 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700727 }
728
729 bool WaitForFrame(int64_t timeout_ms) {
730 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100731 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700732 return ok;
733 }
734
perkj26091b12016-09-01 01:17:40 -0700735 class TestEncoder : public test::FakeEncoder {
736 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100737 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700738
asaperssonfab67072017-04-04 05:51:49 -0700739 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800740 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700741 return config_;
742 }
743
744 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800745 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700746 block_next_encode_ = true;
747 }
748
Erik Språngaed30702018-11-05 12:57:17 +0100749 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800750 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100751 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100752 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100753 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100754 info.scaling_settings = VideoEncoder::ScalingSettings(
755 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100756 }
757 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100758 for (int i = 0; i < kMaxSpatialLayers; ++i) {
759 if (temporal_layers_supported_[i]) {
760 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
761 info.fps_allocation[i].resize(num_layers);
762 }
763 }
Erik Språngaed30702018-11-05 12:57:17 +0100764 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200765
766 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100767 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100768 return info;
kthelgason876222f2016-11-29 01:44:11 -0800769 }
770
Erik Språngb7cb7b52019-02-26 15:52:33 +0100771 int32_t RegisterEncodeCompleteCallback(
772 EncodedImageCallback* callback) override {
773 rtc::CritScope lock(&local_crit_sect_);
774 encoded_image_callback_ = callback;
775 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
776 }
777
perkjfa10b552016-10-02 23:45:26 -0700778 void ContinueEncode() { continue_encode_event_.Set(); }
779
780 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
781 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800782 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700783 EXPECT_EQ(timestamp_, timestamp);
784 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
785 }
786
kthelgason2fc52542017-03-03 00:24:41 -0800787 void SetQualityScaling(bool b) {
788 rtc::CritScope lock(&local_crit_sect_);
789 quality_scaling_ = b;
790 }
kthelgasonad9010c2017-02-14 00:46:51 -0800791
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100792 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
793 rtc::CritScope lock(&local_crit_sect_);
794 requested_resolution_alignment_ = requested_resolution_alignment;
795 }
796
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100797 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
798 rtc::CritScope lock(&local_crit_sect_);
799 is_hardware_accelerated_ = is_hardware_accelerated;
800 }
801
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100802 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
803 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
804 rtc::CritScope lock(&local_crit_sect_);
805 temporal_layers_supported_[spatial_idx] = supported;
806 }
807
Sergey Silkin6456e352019-07-08 17:56:40 +0200808 void SetResolutionBitrateLimits(
809 std::vector<ResolutionBitrateLimits> thresholds) {
810 rtc::CritScope cs(&local_crit_sect_);
811 resolution_bitrate_limits_ = thresholds;
812 }
813
sprangfe627f32017-03-29 08:24:59 -0700814 void ForceInitEncodeFailure(bool force_failure) {
815 rtc::CritScope lock(&local_crit_sect_);
816 force_init_encode_failed_ = force_failure;
817 }
818
Niels Möller6bb5ab92019-01-11 11:11:10 +0100819 void SimulateOvershoot(double rate_factor) {
820 rtc::CritScope lock(&local_crit_sect_);
821 rate_factor_ = rate_factor;
822 }
823
Erik Språngd7329ca2019-02-21 21:19:53 +0100824 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100825 rtc::CritScope lock(&local_crit_sect_);
826 return last_framerate_;
827 }
828
Erik Språngd7329ca2019-02-21 21:19:53 +0100829 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100830 rtc::CritScope lock(&local_crit_sect_);
831 return last_update_rect_;
832 }
833
Niels Möller87e2d782019-03-07 10:18:23 +0100834 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100835 rtc::CritScope lock(&local_crit_sect_);
836 return last_frame_types_;
837 }
838
839 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100840 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100841 keyframe ? VideoFrameType::kVideoFrameKey
842 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100843 {
844 rtc::CritScope lock(&local_crit_sect_);
845 last_frame_types_ = frame_type;
846 }
Niels Möllerb859b322019-03-07 12:40:01 +0100847 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100848 }
849
Erik Språngb7cb7b52019-02-26 15:52:33 +0100850 void InjectEncodedImage(const EncodedImage& image) {
851 rtc::CritScope lock(&local_crit_sect_);
852 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
853 }
854
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200855 void InjectEncodedImage(const EncodedImage& image,
856 const CodecSpecificInfo* codec_specific_info,
857 const RTPFragmentationHeader* fragmentation) {
858 rtc::CritScope lock(&local_crit_sect_);
859 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
860 fragmentation);
861 }
862
Erik Språngd7329ca2019-02-21 21:19:53 +0100863 void ExpectNullFrame() {
864 rtc::CritScope lock(&local_crit_sect_);
865 expect_null_frame_ = true;
866 }
867
Erik Språng5056af02019-09-02 15:53:11 +0200868 absl::optional<VideoEncoder::RateControlParameters>
869 GetAndResetLastRateControlSettings() {
870 auto settings = last_rate_control_settings_;
871 last_rate_control_settings_.reset();
872 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100873 }
874
Sergey Silkin5ee69672019-07-02 14:18:34 +0200875 int GetNumEncoderInitializations() const {
876 rtc::CritScope lock(&local_crit_sect_);
877 return num_encoder_initializations_;
878 }
879
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200880 int GetNumSetRates() const {
881 rtc::CritScope lock(&local_crit_sect_);
882 return num_set_rates_;
883 }
884
perkjfa10b552016-10-02 23:45:26 -0700885 private:
perkj26091b12016-09-01 01:17:40 -0700886 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100887 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700888 bool block_encode;
889 {
brandtre78d2662017-01-16 05:57:16 -0800890 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100891 if (expect_null_frame_) {
892 EXPECT_EQ(input_image.timestamp(), 0u);
893 EXPECT_EQ(input_image.width(), 1);
894 last_frame_types_ = *frame_types;
895 expect_null_frame_ = false;
896 } else {
897 EXPECT_GT(input_image.timestamp(), timestamp_);
898 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
899 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
900 }
perkj26091b12016-09-01 01:17:40 -0700901
902 timestamp_ = input_image.timestamp();
903 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700904 last_input_width_ = input_image.width();
905 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700906 block_encode = block_next_encode_;
907 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100908 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100909 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700910 }
Niels Möllerb859b322019-03-07 12:40:01 +0100911 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700912 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700913 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700914 return result;
915 }
916
sprangfe627f32017-03-29 08:24:59 -0700917 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200918 const Settings& settings) override {
919 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200920
sprangfe627f32017-03-29 08:24:59 -0700921 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100922 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200923
924 ++num_encoder_initializations_;
925
Erik Språng82fad3d2018-03-21 09:57:23 +0100926 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700927 // Simulate setting up temporal layers, in order to validate the life
928 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100929 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200930 frame_buffer_controller_ =
931 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700932 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100933 if (force_init_encode_failed_) {
934 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700935 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100936 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100937
Erik Språngb7cb7b52019-02-26 15:52:33 +0100938 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -0700939 return res;
940 }
941
Erik Språngb7cb7b52019-02-26 15:52:33 +0100942 int32_t Release() override {
943 rtc::CritScope lock(&local_crit_sect_);
944 EXPECT_NE(initialized_, EncoderState::kUninitialized);
945 initialized_ = EncoderState::kUninitialized;
946 return FakeEncoder::Release();
947 }
948
Erik Språng16cb8f52019-04-12 13:59:09 +0200949 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100950 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200951 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100952 VideoBitrateAllocation adjusted_rate_allocation;
953 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
954 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +0200955 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100956 adjusted_rate_allocation.SetBitrate(
957 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +0200958 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +0100959 rate_factor_));
960 }
961 }
962 }
Erik Språng16cb8f52019-04-12 13:59:09 +0200963 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +0200964 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +0200965 RateControlParameters adjusted_paramters = parameters;
966 adjusted_paramters.bitrate = adjusted_rate_allocation;
967 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100968 }
969
brandtre78d2662017-01-16 05:57:16 -0800970 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100971 enum class EncoderState {
972 kUninitialized,
973 kInitializationFailed,
974 kInitialized
975 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
976 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -0700977 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -0700978 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -0700979 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
980 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
981 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
982 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
983 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100984 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100985 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +0100986 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -0700987 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100988 absl::optional<bool>
989 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
990 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -0700991 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +0100992 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
993 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +0200994 absl::optional<VideoEncoder::RateControlParameters>
995 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100996 VideoFrame::UpdateRect last_update_rect_
997 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +0100998 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +0100999 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001000 EncodedImageCallback* encoded_image_callback_
1001 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Elad Alon45befc52019-07-02 11:20:09 +02001002 MockFecControllerOverride fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001003 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001004 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1005 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001006 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001007 };
1008
mflodmancc3d4422017-08-03 08:27:51 -07001009 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001010 public:
1011 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001012 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001013
perkj26091b12016-09-01 01:17:40 -07001014 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001015 EXPECT_TRUE(
1016 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1017 }
1018
1019 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1020 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001021 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001022 if (!encoded_frame_event_.Wait(timeout_ms))
1023 return false;
perkj26091b12016-09-01 01:17:40 -07001024 {
1025 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001026 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001027 }
1028 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001029 return true;
perkj26091b12016-09-01 01:17:40 -07001030 }
1031
sprangb1ca0732017-02-01 08:38:12 -08001032 void WaitForEncodedFrame(uint32_t expected_width,
1033 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001034 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001035 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001036 }
1037
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001038 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001039 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001040 uint32_t width = 0;
1041 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001042 {
1043 rtc::CritScope lock(&crit_);
1044 width = last_width_;
1045 height = last_height_;
1046 }
1047 EXPECT_EQ(expected_height, height);
1048 EXPECT_EQ(expected_width, width);
1049 }
1050
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001051 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1052 int width = 0;
1053 int height = 0;
1054 {
1055 rtc::CritScope lock(&crit_);
1056 width = last_width_;
1057 height = last_height_;
1058 }
1059 EXPECT_EQ(width % resolution_alignment, 0);
1060 EXPECT_EQ(height % resolution_alignment, 0);
1061 }
1062
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001063 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1064 VideoRotation rotation;
1065 {
1066 rtc::CritScope lock(&crit_);
1067 rotation = last_rotation_;
1068 }
1069 EXPECT_EQ(expected_rotation, rotation);
1070 }
1071
kthelgason2fc52542017-03-03 00:24:41 -08001072 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001073
sprangc5d62e22017-04-02 23:53:04 -07001074 bool WaitForFrame(int64_t timeout_ms) {
1075 return encoded_frame_event_.Wait(timeout_ms);
1076 }
1077
perkj26091b12016-09-01 01:17:40 -07001078 void SetExpectNoFrames() {
1079 rtc::CritScope lock(&crit_);
1080 expect_frames_ = false;
1081 }
1082
asaperssonfab67072017-04-04 05:51:49 -07001083 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001084 rtc::CritScope lock(&crit_);
1085 return number_of_reconfigurations_;
1086 }
1087
asaperssonfab67072017-04-04 05:51:49 -07001088 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001089 rtc::CritScope lock(&crit_);
1090 return min_transmit_bitrate_bps_;
1091 }
1092
Erik Språngd7329ca2019-02-21 21:19:53 +01001093 void SetNumExpectedLayers(size_t num_layers) {
1094 rtc::CritScope lock(&crit_);
1095 num_expected_layers_ = num_layers;
1096 }
1097
Erik Språngb7cb7b52019-02-26 15:52:33 +01001098 int64_t GetLastCaptureTimeMs() const {
1099 rtc::CritScope lock(&crit_);
1100 return last_capture_time_ms_;
1101 }
1102
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001103 std::vector<uint8_t> GetLastEncodedImageData() {
1104 rtc::CritScope lock(&crit_);
1105 return std::move(last_encoded_image_data_);
1106 }
1107
1108 RTPFragmentationHeader GetLastFragmentation() {
1109 rtc::CritScope lock(&crit_);
1110 return std::move(last_fragmentation_);
1111 }
1112
perkj26091b12016-09-01 01:17:40 -07001113 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001114 Result OnEncodedImage(
1115 const EncodedImage& encoded_image,
1116 const CodecSpecificInfo* codec_specific_info,
1117 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001118 rtc::CritScope lock(&crit_);
1119 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001120 last_encoded_image_data_ = std::vector<uint8_t>(
1121 encoded_image.data(), encoded_image.data() + encoded_image.size());
1122 if (fragmentation) {
1123 last_fragmentation_.CopyFrom(*fragmentation);
1124 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001125 uint32_t timestamp = encoded_image.Timestamp();
1126 if (last_timestamp_ != timestamp) {
1127 num_received_layers_ = 1;
1128 } else {
1129 ++num_received_layers_;
1130 }
1131 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001132 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001133 last_width_ = encoded_image._encodedWidth;
1134 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001135 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001136 if (num_received_layers_ == num_expected_layers_) {
1137 encoded_frame_event_.Set();
1138 }
sprangb1ca0732017-02-01 08:38:12 -08001139 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001140 }
1141
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001142 void OnEncoderConfigurationChanged(
1143 std::vector<VideoStream> streams,
1144 VideoEncoderConfig::ContentType content_type,
1145 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001146 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001147 ++number_of_reconfigurations_;
1148 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1149 }
1150
perkj26091b12016-09-01 01:17:40 -07001151 rtc::CriticalSection crit_;
1152 TestEncoder* test_encoder_;
1153 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001154 std::vector<uint8_t> last_encoded_image_data_;
1155 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001156 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001157 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001158 uint32_t last_height_ = 0;
1159 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001160 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001161 size_t num_expected_layers_ = 1;
1162 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001163 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001164 int number_of_reconfigurations_ = 0;
1165 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001166 };
1167
Sergey Silkin5ee69672019-07-02 14:18:34 +02001168 class VideoBitrateAllocatorProxyFactory
1169 : public VideoBitrateAllocatorFactory {
1170 public:
1171 VideoBitrateAllocatorProxyFactory()
1172 : bitrate_allocator_factory_(
1173 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1174
1175 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1176 const VideoCodec& codec) override {
1177 rtc::CritScope lock(&crit_);
1178 codec_config_ = codec;
1179 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1180 }
1181
1182 VideoCodec codec_config() const {
1183 rtc::CritScope lock(&crit_);
1184 return codec_config_;
1185 }
1186
1187 private:
1188 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1189
1190 rtc::CriticalSection crit_;
1191 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1192 };
1193
perkj26091b12016-09-01 01:17:40 -07001194 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001195 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001196 int codec_width_;
1197 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001198 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001199 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001200 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001201 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001202 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001203 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001204 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001205 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001206 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001207 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001208};
1209
mflodmancc3d4422017-08-03 08:27:51 -07001210TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001211 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001212 DataRate::BitsPerSec(kTargetBitrateBps),
1213 DataRate::BitsPerSec(kTargetBitrateBps),
1214 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001215 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001216 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001217 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001218 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001219 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001220}
1221
mflodmancc3d4422017-08-03 08:27:51 -07001222TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001223 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001224 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001225 // The encoder will cache up to one frame for a short duration. Adding two
1226 // frames means that the first frame will be dropped and the second frame will
1227 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001228 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001229 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001230 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001231
Erik Språng4c6ca302019-04-08 15:14:01 +02001232 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001233 DataRate::BitsPerSec(kTargetBitrateBps),
1234 DataRate::BitsPerSec(kTargetBitrateBps),
1235 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001236
Sebastian Janssona3177052018-04-10 13:05:49 +02001237 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001238 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001239 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1240
1241 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001242 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001243}
1244
mflodmancc3d4422017-08-03 08:27:51 -07001245TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001246 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001247 DataRate::BitsPerSec(kTargetBitrateBps),
1248 DataRate::BitsPerSec(kTargetBitrateBps),
1249 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001250 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001251 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001252
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001253 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
1254 DataRate::BitsPerSec(0),
1255 DataRate::BitsPerSec(0), 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001256 // The encoder will cache up to one frame for a short duration. Adding two
1257 // frames means that the first frame will be dropped and the second frame will
1258 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001259 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001260 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001261
Erik Språng4c6ca302019-04-08 15:14:01 +02001262 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001263 DataRate::BitsPerSec(kTargetBitrateBps),
1264 DataRate::BitsPerSec(kTargetBitrateBps),
1265 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001266 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001267 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1268 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001269 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001270}
1271
mflodmancc3d4422017-08-03 08:27:51 -07001272TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001273 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001274 DataRate::BitsPerSec(kTargetBitrateBps),
1275 DataRate::BitsPerSec(kTargetBitrateBps),
1276 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001277 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001278 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001279
1280 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001281 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001282
perkja49cbd32016-09-16 07:53:41 -07001283 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001284 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001285 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001286}
1287
mflodmancc3d4422017-08-03 08:27:51 -07001288TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001289 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001290 DataRate::BitsPerSec(kTargetBitrateBps),
1291 DataRate::BitsPerSec(kTargetBitrateBps),
1292 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001293
perkja49cbd32016-09-16 07:53:41 -07001294 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001295 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001296
mflodmancc3d4422017-08-03 08:27:51 -07001297 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001298 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001299 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001300 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1301 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001302}
1303
mflodmancc3d4422017-08-03 08:27:51 -07001304TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001305 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001306 DataRate::BitsPerSec(kTargetBitrateBps),
1307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001309
1310 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001311 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001312 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001313 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1314 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001315 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1316 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001317 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001318 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001319
mflodmancc3d4422017-08-03 08:27:51 -07001320 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001321}
1322
Noah Richards51db4212019-06-12 06:59:12 -07001323TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
1324 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001325 DataRate::BitsPerSec(kTargetBitrateBps),
1326 DataRate::BitsPerSec(kTargetBitrateBps),
1327 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001328
1329 rtc::Event frame_destroyed_event;
1330 video_source_.IncomingCapturedFrame(
1331 CreateFakeNativeFrame(1, &frame_destroyed_event));
1332 ExpectDroppedFrame();
1333 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1334 video_stream_encoder_->Stop();
1335}
1336
1337TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1338 // Use the cropping factory.
1339 video_encoder_config_.video_stream_factory =
1340 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1341 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1342 kMaxPayloadLength);
1343 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1344
1345 // Capture a frame at codec_width_/codec_height_.
1346 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001347 DataRate::BitsPerSec(kTargetBitrateBps),
1348 DataRate::BitsPerSec(kTargetBitrateBps),
1349 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001350 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1351 WaitForEncodedFrame(1);
1352 // The encoder will have been configured once.
1353 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1354 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1355 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1356
1357 // Now send in a fake frame that needs to be cropped as the width/height
1358 // aren't divisible by 4 (see CreateEncoderStreams above).
1359 rtc::Event frame_destroyed_event;
1360 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1361 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1362 ExpectDroppedFrame();
1363 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1364 video_stream_encoder_->Stop();
1365}
1366
Ying Wang9b881ab2020-02-07 14:29:32 +01001367TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
1368 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001369 DataRate::BitsPerSec(kTargetBitrateBps),
1370 DataRate::BitsPerSec(kTargetBitrateBps),
1371 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001372 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1373 WaitForEncodedFrame(1);
1374
1375 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001376 DataRate::BitsPerSec(kTargetBitrateBps),
1377 DataRate::BitsPerSec(kTargetBitrateBps),
1378 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001379 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1380 // frames. Adding two frames means that the first frame will be dropped and
1381 // the second frame will be sent to the encoder.
1382 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1383 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1384 WaitForEncodedFrame(3);
1385 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1386 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1387 WaitForEncodedFrame(5);
1388 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1389 video_stream_encoder_->Stop();
1390}
1391
mflodmancc3d4422017-08-03 08:27:51 -07001392TEST_F(VideoStreamEncoderTest,
1393 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001394 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001395 DataRate::BitsPerSec(kTargetBitrateBps),
1396 DataRate::BitsPerSec(kTargetBitrateBps),
1397 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001398 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001399
1400 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001401 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001402 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001403 // The encoder will have been configured once when the first frame is
1404 // received.
1405 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001406
1407 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001408 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001409 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001410 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001411 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001412
1413 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001414 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001415 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001416 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001417 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001418
mflodmancc3d4422017-08-03 08:27:51 -07001419 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001420}
1421
mflodmancc3d4422017-08-03 08:27:51 -07001422TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001423 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001424 DataRate::BitsPerSec(kTargetBitrateBps),
1425 DataRate::BitsPerSec(kTargetBitrateBps),
1426 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001427
1428 // Capture a frame and wait for it to synchronize with the encoder thread.
1429 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001430 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001431 // The encoder will have been configured once.
1432 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001433 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1434 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1435
1436 codec_width_ *= 2;
1437 codec_height_ *= 2;
1438 // Capture a frame with a higher resolution and wait for it to synchronize
1439 // with the encoder thread.
1440 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001441 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001442 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1443 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001444 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001445
mflodmancc3d4422017-08-03 08:27:51 -07001446 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001447}
1448
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001449TEST_F(VideoStreamEncoderTest,
1450 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
1451 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001452 DataRate::BitsPerSec(kTargetBitrateBps),
1453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001455
1456 // Capture a frame and wait for it to synchronize with the encoder thread.
1457 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1458 WaitForEncodedFrame(1);
1459
1460 VideoEncoderConfig video_encoder_config;
1461 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1462 // Changing the max payload data length recreates encoder.
1463 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1464 kMaxPayloadLength / 2);
1465
1466 // Capture a frame and wait for it to synchronize with the encoder thread.
1467 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1468 WaitForEncodedFrame(2);
1469 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1470
1471 video_stream_encoder_->Stop();
1472}
1473
Sergey Silkin5ee69672019-07-02 14:18:34 +02001474TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
1475 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001476 DataRate::BitsPerSec(kTargetBitrateBps),
1477 DataRate::BitsPerSec(kTargetBitrateBps),
1478 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001479
1480 VideoEncoderConfig video_encoder_config;
1481 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1482 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1483 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1484 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1485 kMaxPayloadLength);
1486
1487 // Capture a frame and wait for it to synchronize with the encoder thread.
1488 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1489 WaitForEncodedFrame(1);
1490 // The encoder will have been configured once when the first frame is
1491 // received.
1492 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1493 EXPECT_EQ(kTargetBitrateBps,
1494 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1495 EXPECT_EQ(kStartBitrateBps,
1496 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1497
Sergey Silkin6456e352019-07-08 17:56:40 +02001498 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1499 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001500 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1501 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1502 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1503 kMaxPayloadLength);
1504
1505 // Capture a frame and wait for it to synchronize with the encoder thread.
1506 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1507 WaitForEncodedFrame(2);
1508 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1509 // Bitrate limits have changed - rate allocator should be reconfigured,
1510 // encoder should not be reconfigured.
1511 EXPECT_EQ(kTargetBitrateBps * 2,
1512 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1513 EXPECT_EQ(kStartBitrateBps * 2,
1514 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1515 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1516
1517 video_stream_encoder_->Stop();
1518}
1519
Sergey Silkin6456e352019-07-08 17:56:40 +02001520TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001521 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001522 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001523 DataRate::BitsPerSec(kTargetBitrateBps),
1524 DataRate::BitsPerSec(kTargetBitrateBps),
1525 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001526
Sergey Silkincd02eba2020-01-20 14:48:40 +01001527 const uint32_t kMinEncBitrateKbps = 100;
1528 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001529 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001530 /*frame_size_pixels=*/codec_width_ * codec_height_,
1531 /*min_start_bitrate_bps=*/0,
1532 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1533 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001534 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1535
Sergey Silkincd02eba2020-01-20 14:48:40 +01001536 VideoEncoderConfig video_encoder_config;
1537 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1538 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1539 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1540 (kMinEncBitrateKbps + 1) * 1000;
1541 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1542 kMaxPayloadLength);
1543
1544 // When both encoder and app provide bitrate limits, the intersection of
1545 // provided sets should be used.
1546 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1547 WaitForEncodedFrame(1);
1548 EXPECT_EQ(kMaxEncBitrateKbps,
1549 bitrate_allocator_factory_.codec_config().maxBitrate);
1550 EXPECT_EQ(kMinEncBitrateKbps + 1,
1551 bitrate_allocator_factory_.codec_config().minBitrate);
1552
1553 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1554 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1555 (kMinEncBitrateKbps - 1) * 1000;
1556 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1557 kMaxPayloadLength);
1558 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001559 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001560 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001561 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001562 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001563 bitrate_allocator_factory_.codec_config().minBitrate);
1564
Sergey Silkincd02eba2020-01-20 14:48:40 +01001565 video_stream_encoder_->Stop();
1566}
1567
1568TEST_F(VideoStreamEncoderTest,
1569 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
1570 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001571 DataRate::BitsPerSec(kTargetBitrateBps),
1572 DataRate::BitsPerSec(kTargetBitrateBps),
1573 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001574
1575 const uint32_t kMinAppBitrateKbps = 100;
1576 const uint32_t kMaxAppBitrateKbps = 200;
1577 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1578 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1579 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1580 /*frame_size_pixels=*/codec_width_ * codec_height_,
1581 /*min_start_bitrate_bps=*/0,
1582 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1583 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1584 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1585
1586 VideoEncoderConfig video_encoder_config;
1587 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1588 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1589 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1590 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001591 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1592 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001593
Sergey Silkincd02eba2020-01-20 14:48:40 +01001594 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1595 WaitForEncodedFrame(1);
1596 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001597 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001598 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001599 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001600
1601 video_stream_encoder_->Stop();
1602}
1603
1604TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001605 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Sergey Silkin6456e352019-07-08 17:56:40 +02001606 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001607 DataRate::BitsPerSec(kTargetBitrateBps),
1608 DataRate::BitsPerSec(kTargetBitrateBps),
1609 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001610
1611 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001612 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001613 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001614 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001615 fake_encoder_.SetResolutionBitrateLimits(
1616 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1617
1618 VideoEncoderConfig video_encoder_config;
1619 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1620 video_encoder_config.max_bitrate_bps = 0;
1621 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1622 kMaxPayloadLength);
1623
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001624 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001625 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1626 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001627 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1628 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001629 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1630 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1631
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001632 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001633 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1634 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001635 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1636 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001637 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1638 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1639
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001640 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001641 // encoder for 360p should be used.
1642 video_source_.IncomingCapturedFrame(
1643 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1644 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001645 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1646 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001647 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1648 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1649
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001650 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001651 // ignored.
1652 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1653 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001654 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1655 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001656 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1657 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001658 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1659 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001660 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1661 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1662
1663 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1664 // for 270p should be used.
1665 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1666 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001667 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1668 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001669 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1670 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1671
1672 video_stream_encoder_->Stop();
1673}
1674
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001675TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
1676 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001677 DataRate::BitsPerSec(kTargetBitrateBps),
1678 DataRate::BitsPerSec(kTargetBitrateBps),
1679 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001680
1681 VideoEncoderConfig video_encoder_config;
1682 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1683 video_encoder_config.max_bitrate_bps = 0;
1684 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1685 kMaxPayloadLength);
1686
1687 // Encode 720p frame to get the default encoder target bitrate.
1688 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1689 WaitForEncodedFrame(1);
1690 const uint32_t kDefaultTargetBitrateFor720pKbps =
1691 bitrate_allocator_factory_.codec_config()
1692 .simulcastStream[0]
1693 .targetBitrate;
1694
1695 // Set the max recommended encoder bitrate to something lower than the default
1696 // target bitrate.
1697 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1698 1280 * 720, 10 * 1000, 10 * 1000,
1699 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1700 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1701
1702 // Change resolution to trigger encoder reinitialization.
1703 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1704 WaitForEncodedFrame(2);
1705 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1706 WaitForEncodedFrame(3);
1707
1708 // Ensure the target bitrate is capped by the max bitrate.
1709 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1710 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1711 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1712 .simulcastStream[0]
1713 .targetBitrate *
1714 1000,
1715 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1716
1717 video_stream_encoder_->Stop();
1718}
1719
mflodmancc3d4422017-08-03 08:27:51 -07001720TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001721 EXPECT_TRUE(video_source_.has_sinks());
1722 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001723 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001724 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001725 EXPECT_FALSE(video_source_.has_sinks());
1726 EXPECT_TRUE(new_video_source.has_sinks());
1727
mflodmancc3d4422017-08-03 08:27:51 -07001728 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001729}
1730
mflodmancc3d4422017-08-03 08:27:51 -07001731TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001732 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001733 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001734 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001735 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001736}
1737
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001738TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1739 constexpr int kRequestedResolutionAlignment = 7;
1740 video_source_.set_adaptation_enabled(true);
1741 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
1742 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001743 DataRate::BitsPerSec(kTargetBitrateBps),
1744 DataRate::BitsPerSec(kTargetBitrateBps),
1745 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001746
1747 // On the 1st frame, we should have initialized the encoder and
1748 // asked for its resolution requirements.
1749 video_source_.IncomingCapturedFrame(
1750 CreateFrame(1, codec_width_, codec_height_));
1751 WaitForEncodedFrame(1);
1752 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1753 kRequestedResolutionAlignment);
1754
1755 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1756 // (It's up the to the encoder to potentially drop the previous frame,
1757 // to avoid coding back-to-back keyframes.)
1758 video_source_.IncomingCapturedFrame(
1759 CreateFrame(2, codec_width_, codec_height_));
1760 WaitForEncodedFrame(2);
1761 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1762
1763 video_stream_encoder_->Stop();
1764}
1765
Jonathan Yubc771b72017-12-08 17:04:29 -08001766TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1767 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001768 const int kWidth = 1280;
1769 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001770
1771 // We rely on the automatic resolution adaptation, but we handle framerate
1772 // adaptation manually by mocking the stats proxy.
1773 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001774
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001775 // Enable BALANCED preference, no initial limitation.
Erik Språng4c6ca302019-04-08 15:14:01 +02001776 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001777 DataRate::BitsPerSec(kTargetBitrateBps),
1778 DataRate::BitsPerSec(kTargetBitrateBps),
1779 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001780 video_stream_encoder_->SetSource(&video_source_,
1781 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001782 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001785 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1786
Jonathan Yubc771b72017-12-08 17:04:29 -08001787 // Adapt down as far as possible.
1788 rtc::VideoSinkWants last_wants;
1789 int64_t t = 1;
1790 int loop_count = 0;
1791 do {
1792 ++loop_count;
1793 last_wants = video_source_.sink_wants();
1794
1795 // Simulate the framerate we've been asked to adapt to.
1796 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1797 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1798 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1799 mock_stats.input_frame_rate = fps;
1800 stats_proxy_->SetMockStats(mock_stats);
1801
1802 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1803 sink_.WaitForEncodedFrame(t);
1804 t += frame_interval_ms;
1805
mflodmancc3d4422017-08-03 08:27:51 -07001806 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001807 VerifyBalancedModeFpsRange(
1808 video_source_.sink_wants(),
1809 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1810 } while (video_source_.sink_wants().max_pixel_count <
1811 last_wants.max_pixel_count ||
1812 video_source_.sink_wants().max_framerate_fps <
1813 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001814
Jonathan Yubc771b72017-12-08 17:04:29 -08001815 // Verify that we've adapted all the way down.
1816 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001817 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001818 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1819 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001820 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001821 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1822 *video_source_.last_sent_height());
1823 EXPECT_EQ(kMinBalancedFramerateFps,
1824 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001825
Jonathan Yubc771b72017-12-08 17:04:29 -08001826 // Adapt back up the same number of times we adapted down.
1827 for (int i = 0; i < loop_count - 1; ++i) {
1828 last_wants = video_source_.sink_wants();
1829
1830 // Simulate the framerate we've been asked to adapt to.
1831 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1832 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1833 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1834 mock_stats.input_frame_rate = fps;
1835 stats_proxy_->SetMockStats(mock_stats);
1836
1837 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1838 sink_.WaitForEncodedFrame(t);
1839 t += frame_interval_ms;
1840
mflodmancc3d4422017-08-03 08:27:51 -07001841 video_stream_encoder_->TriggerCpuNormalUsage();
Jonathan Yubc771b72017-12-08 17:04:29 -08001842 VerifyBalancedModeFpsRange(
1843 video_source_.sink_wants(),
1844 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1845 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1846 last_wants.max_pixel_count ||
1847 video_source_.sink_wants().max_framerate_fps >
1848 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001849 }
1850
Åsa Persson8c1bf952018-09-13 10:42:19 +02001851 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001852 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1855 EXPECT_EQ((loop_count - 1) * 2,
1856 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001857
mflodmancc3d4422017-08-03 08:27:51 -07001858 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001859}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001860
mflodmancc3d4422017-08-03 08:27:51 -07001861TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001862 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001863 DataRate::BitsPerSec(kTargetBitrateBps),
1864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07001866 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001867
sprangc5d62e22017-04-02 23:53:04 -07001868 const int kFrameWidth = 1280;
1869 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07001870
Åsa Persson8c1bf952018-09-13 10:42:19 +02001871 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07001872
kthelgason5e13d412016-12-01 03:59:51 -08001873 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001874 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001875 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001876 frame_timestamp += kFrameIntervalMs;
1877
perkj803d97f2016-11-01 11:45:46 -07001878 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07001879 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001880 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001881 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001882 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001883 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07001884
asapersson0944a802017-04-07 00:57:58 -07001885 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07001886 // wanted resolution.
1887 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1888 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1889 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001890 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001891
1892 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07001893 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001894 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001895 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001896 // Give the encoder queue time to process the change in degradation preference
1897 // by waiting for an encoded frame.
1898 new_video_source.IncomingCapturedFrame(
1899 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1900 sink_.WaitForEncodedFrame(frame_timestamp);
1901 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001902 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001903 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001904
sprangc5d62e22017-04-02 23:53:04 -07001905 // Force an input frame rate to be available, or the adaptation call won't
1906 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07001907 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07001908 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07001909 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07001910 stats_proxy_->SetMockStats(stats);
1911
mflodmancc3d4422017-08-03 08:27:51 -07001912 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07001913 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07001914 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001915 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001916 frame_timestamp += kFrameIntervalMs;
1917
1918 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08001919 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07001920 EXPECT_EQ(std::numeric_limits<int>::max(),
1921 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001922 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07001923
asapersson02465b82017-04-10 01:12:52 -07001924 // Turn off degradation completely.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001925 video_stream_encoder_->SetSource(&new_video_source,
1926 webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01001927 // Give the encoder queue time to process the change in degradation preference
1928 // by waiting for an encoded frame.
1929 new_video_source.IncomingCapturedFrame(
1930 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1931 sink_.WaitForEncodedFrame(frame_timestamp);
1932 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02001933 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07001934
mflodmancc3d4422017-08-03 08:27:51 -07001935 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07001936 new_video_source.IncomingCapturedFrame(
1937 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07001938 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07001939 frame_timestamp += kFrameIntervalMs;
1940
1941 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02001942 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07001943
1944 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001945 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001946 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01001947 // Give the encoder queue time to process the change in degradation preference
1948 // by waiting for an encoded frame.
1949 new_video_source.IncomingCapturedFrame(
1950 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1951 sink_.WaitForEncodedFrame(frame_timestamp);
1952 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001953 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
1954 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08001955 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02001956 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07001957
1958 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
mflodmancc3d4422017-08-03 08:27:51 -07001959 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001960 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01001961 // Give the encoder queue time to process the change in degradation preference
1962 // by waiting for an encoded frame.
1963 new_video_source.IncomingCapturedFrame(
1964 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
1965 sink_.WaitForEncodedFrame(frame_timestamp);
1966 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07001967 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
1968 EXPECT_EQ(std::numeric_limits<int>::max(),
1969 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07001970 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07001971
mflodmancc3d4422017-08-03 08:27:51 -07001972 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001973}
1974
mflodmancc3d4422017-08-03 08:27:51 -07001975TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02001976 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001977 DataRate::BitsPerSec(kTargetBitrateBps),
1978 DataRate::BitsPerSec(kTargetBitrateBps),
1979 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07001980
asaperssonfab67072017-04-04 05:51:49 -07001981 const int kWidth = 1280;
1982 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07001983 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001984 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07001985 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1986 EXPECT_FALSE(stats.bw_limited_resolution);
1987 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
1988
1989 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07001990 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07001991 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07001992 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07001993
1994 stats = stats_proxy_->GetStats();
1995 EXPECT_TRUE(stats.bw_limited_resolution);
1996 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
1997
1998 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07001999 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002000 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002001 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002002
2003 stats = stats_proxy_->GetStats();
2004 EXPECT_FALSE(stats.bw_limited_resolution);
2005 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2006 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2007
mflodmancc3d4422017-08-03 08:27:51 -07002008 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002009}
2010
mflodmancc3d4422017-08-03 08:27:51 -07002011TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002012 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002013 DataRate::BitsPerSec(kTargetBitrateBps),
2014 DataRate::BitsPerSec(kTargetBitrateBps),
2015 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002016
2017 const int kWidth = 1280;
2018 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002019 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002020 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002021 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2022 EXPECT_FALSE(stats.cpu_limited_resolution);
2023 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2024
2025 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002026 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002027 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002028 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002029
2030 stats = stats_proxy_->GetStats();
2031 EXPECT_TRUE(stats.cpu_limited_resolution);
2032 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2033
2034 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07002035 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002036 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002037 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002038
2039 stats = stats_proxy_->GetStats();
2040 EXPECT_FALSE(stats.cpu_limited_resolution);
2041 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002042 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002043
mflodmancc3d4422017-08-03 08:27:51 -07002044 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002045}
2046
mflodmancc3d4422017-08-03 08:27:51 -07002047TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002048 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002049 DataRate::BitsPerSec(kTargetBitrateBps),
2050 DataRate::BitsPerSec(kTargetBitrateBps),
2051 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002052
asaperssonfab67072017-04-04 05:51:49 -07002053 const int kWidth = 1280;
2054 const int kHeight = 720;
2055 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002056 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002057 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002058 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002059 EXPECT_FALSE(stats.cpu_limited_resolution);
2060 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2061
asaperssonfab67072017-04-04 05:51:49 -07002062 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002063 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002064 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002065 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002066 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002067 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002068 EXPECT_TRUE(stats.cpu_limited_resolution);
2069 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2070
2071 // Set new source with adaptation still enabled.
2072 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002073 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002074 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002075
asaperssonfab67072017-04-04 05:51:49 -07002076 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002077 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002078 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002079 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002080 EXPECT_TRUE(stats.cpu_limited_resolution);
2081 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2082
2083 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002084 video_stream_encoder_->SetSource(&new_video_source,
2085 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002086
asaperssonfab67072017-04-04 05:51:49 -07002087 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002088 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002089 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002090 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002091 EXPECT_FALSE(stats.cpu_limited_resolution);
2092 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2093
2094 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002096 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002097
asaperssonfab67072017-04-04 05:51:49 -07002098 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002099 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002100 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002101 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002102 EXPECT_TRUE(stats.cpu_limited_resolution);
2103 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2104
asaperssonfab67072017-04-04 05:51:49 -07002105 // Trigger CPU normal use.
mflodmancc3d4422017-08-03 08:27:51 -07002106 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002107 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002108 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002109 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002110 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002111 EXPECT_FALSE(stats.cpu_limited_resolution);
2112 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002113 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002114
mflodmancc3d4422017-08-03 08:27:51 -07002115 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002116}
2117
mflodmancc3d4422017-08-03 08:27:51 -07002118TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002119 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002120 DataRate::BitsPerSec(kTargetBitrateBps),
2121 DataRate::BitsPerSec(kTargetBitrateBps),
2122 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002123
asaperssonfab67072017-04-04 05:51:49 -07002124 const int kWidth = 1280;
2125 const int kHeight = 720;
2126 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002127 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002128 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002129 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002130 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002131 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002132
2133 // Set new source with adaptation still enabled.
2134 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002135 video_stream_encoder_->SetSource(&new_video_source,
2136 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002137
asaperssonfab67072017-04-04 05:51:49 -07002138 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002139 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002140 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002141 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002142 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002143 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002144
asaperssonfab67072017-04-04 05:51:49 -07002145 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002146 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002147 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002148 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002149 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002150 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002151 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002152 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002153
asaperssonfab67072017-04-04 05:51:49 -07002154 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002155 video_stream_encoder_->SetSource(&new_video_source,
2156 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002157
asaperssonfab67072017-04-04 05:51:49 -07002158 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002159 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002160 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002161 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002162 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002163 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002164
asapersson02465b82017-04-10 01:12:52 -07002165 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002166 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002167 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002168
asaperssonfab67072017-04-04 05:51:49 -07002169 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002170 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002171 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002172 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002173 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002174 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2175 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002176
mflodmancc3d4422017-08-03 08:27:51 -07002177 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002178}
2179
mflodmancc3d4422017-08-03 08:27:51 -07002180TEST_F(VideoStreamEncoderTest,
2181 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002182 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002183 DataRate::BitsPerSec(kTargetBitrateBps),
2184 DataRate::BitsPerSec(kTargetBitrateBps),
2185 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002186
2187 const int kWidth = 1280;
2188 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002189 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002190 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002191 video_source_.IncomingCapturedFrame(
2192 CreateFrame(timestamp_ms, kWidth, kHeight));
2193 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002194 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2195 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2196 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2197
2198 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002199 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002200 timestamp_ms += kFrameIntervalMs;
2201 video_source_.IncomingCapturedFrame(
2202 CreateFrame(timestamp_ms, kWidth, kHeight));
2203 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002204 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2205 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2206 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2207
2208 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002209 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002210 timestamp_ms += kFrameIntervalMs;
2211 video_source_.IncomingCapturedFrame(
2212 CreateFrame(timestamp_ms, kWidth, kHeight));
2213 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002214 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2215 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2216 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2217
Niels Möller4db138e2018-04-19 09:04:13 +02002218 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002219 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002220
2221 VideoEncoderConfig video_encoder_config;
2222 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2223 // Make format different, to force recreation of encoder.
2224 video_encoder_config.video_format.parameters["foo"] = "foo";
2225 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002226 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002227 timestamp_ms += kFrameIntervalMs;
2228 video_source_.IncomingCapturedFrame(
2229 CreateFrame(timestamp_ms, kWidth, kHeight));
2230 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002231 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2232 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2233 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2234
mflodmancc3d4422017-08-03 08:27:51 -07002235 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002236}
2237
mflodmancc3d4422017-08-03 08:27:51 -07002238TEST_F(VideoStreamEncoderTest,
2239 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Erik Språng4c6ca302019-04-08 15:14:01 +02002240 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002241 DataRate::BitsPerSec(kTargetBitrateBps),
2242 DataRate::BitsPerSec(kTargetBitrateBps),
2243 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002244
asapersson0944a802017-04-07 00:57:58 -07002245 const int kWidth = 1280;
2246 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002247 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002248
asaperssonfab67072017-04-04 05:51:49 -07002249 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002250 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002251 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002252 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002253 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002254 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2255
asapersson02465b82017-04-10 01:12:52 -07002256 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002257 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002258 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002259 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002260 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002261 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002262 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002263 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2264
2265 // Set new source with adaptation still enabled.
2266 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002267 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002268 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002269
2270 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002271 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002272 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002273 stats = stats_proxy_->GetStats();
2274 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002275 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002276 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2277
sprangc5d62e22017-04-02 23:53:04 -07002278 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002279 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002280 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002281 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002282 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002283 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002284 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002285 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002286 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002287 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002288 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2289
sprangc5d62e22017-04-02 23:53:04 -07002290 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002291 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002292 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2293 mock_stats.input_frame_rate = 30;
2294 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002295 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002296 stats_proxy_->ResetMockStats();
2297
2298 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002299 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002300 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002301
2302 // Framerate now adapted.
2303 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002304 EXPECT_FALSE(stats.cpu_limited_resolution);
2305 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002306 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2307
2308 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002309 video_stream_encoder_->SetSource(&new_video_source,
2310 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002311 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002312 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002313 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002314
2315 stats = stats_proxy_->GetStats();
2316 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002317 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002318 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2319
2320 // Try to trigger overuse. Should not succeed.
2321 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002322 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002323 stats_proxy_->ResetMockStats();
2324
2325 stats = stats_proxy_->GetStats();
2326 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002327 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002328 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2329
2330 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002331 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002332 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002333 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002334 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002335 stats = stats_proxy_->GetStats();
2336 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002337 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002338 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002339
2340 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002341 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonfab67072017-04-04 05:51:49 -07002342 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002343 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002344 stats = stats_proxy_->GetStats();
2345 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002346 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002347 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2348
2349 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002350 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002351 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002352 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002353 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002354 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002355 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002356 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002357 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002358 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002359 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2360
2361 // Trigger CPU normal usage.
mflodmancc3d4422017-08-03 08:27:51 -07002362 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07002363 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002364 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002365 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002366 stats = stats_proxy_->GetStats();
2367 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002368 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002369 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002370 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002371
mflodmancc3d4422017-08-03 08:27:51 -07002372 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002373}
2374
mflodmancc3d4422017-08-03 08:27:51 -07002375TEST_F(VideoStreamEncoderTest,
2376 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002377 const int kWidth = 1280;
2378 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002379 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002380 DataRate::BitsPerSec(kTargetBitrateBps),
2381 DataRate::BitsPerSec(kTargetBitrateBps),
2382 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002383
asaperssonfab67072017-04-04 05:51:49 -07002384 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002385 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002386
asaperssonfab67072017-04-04 05:51:49 -07002387 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002388 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002389
asaperssonfab67072017-04-04 05:51:49 -07002390 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002391 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002392
asaperssonfab67072017-04-04 05:51:49 -07002393 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002394 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002395
kthelgason876222f2016-11-29 01:44:11 -08002396 // Expect a scale down.
2397 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002398 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002399
asapersson02465b82017-04-10 01:12:52 -07002400 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002401 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002402 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002403 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002404
asaperssonfab67072017-04-04 05:51:49 -07002405 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002406 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002407 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002408 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002409
asaperssonfab67072017-04-04 05:51:49 -07002410 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002411 EXPECT_EQ(std::numeric_limits<int>::max(),
2412 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002413
asaperssonfab67072017-04-04 05:51:49 -07002414 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002415 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002416 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002417 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002418
asapersson02465b82017-04-10 01:12:52 -07002419 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002420 EXPECT_EQ(std::numeric_limits<int>::max(),
2421 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002422
mflodmancc3d4422017-08-03 08:27:51 -07002423 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002424}
2425
mflodmancc3d4422017-08-03 08:27:51 -07002426TEST_F(VideoStreamEncoderTest,
2427 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002428 const int kWidth = 1280;
2429 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002430 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002431 DataRate::BitsPerSec(kTargetBitrateBps),
2432 DataRate::BitsPerSec(kTargetBitrateBps),
2433 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002434
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002435 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002436 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002437 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002438 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002439
2440 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002441 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002442 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002443 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2445
2446 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002447 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002448 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002449 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2450 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2451 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2452
2453 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002454 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002455 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2456 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2457 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2458
mflodmancc3d4422017-08-03 08:27:51 -07002459 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002460}
2461
mflodmancc3d4422017-08-03 08:27:51 -07002462TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002463 const int kWidth = 1280;
2464 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002465 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002466 DataRate::BitsPerSec(kTargetBitrateBps),
2467 DataRate::BitsPerSec(kTargetBitrateBps),
2468 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002469
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002470 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002471 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002472 video_stream_encoder_->SetSource(&source,
2473 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002474 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2475 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002476 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002477
2478 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002479 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002480 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2482 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2483 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2484
2485 // Trigger adapt down for same input resolution, expect no change.
2486 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2487 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002488 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002489 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2490 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2491 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2492
2493 // Trigger adapt down for larger input resolution, expect no change.
2494 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2495 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002496 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002497 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2499 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2500
mflodmancc3d4422017-08-03 08:27:51 -07002501 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002502}
2503
mflodmancc3d4422017-08-03 08:27:51 -07002504TEST_F(VideoStreamEncoderTest,
2505 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002506 const int kWidth = 1280;
2507 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002508 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002509 DataRate::BitsPerSec(kTargetBitrateBps),
2510 DataRate::BitsPerSec(kTargetBitrateBps),
2511 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002512
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002513 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002514 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002515 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002516 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002517
2518 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002519 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002520 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002521 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2522 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2523
2524 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002525 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002526 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002527 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2528 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2529
mflodmancc3d4422017-08-03 08:27:51 -07002530 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002531}
2532
mflodmancc3d4422017-08-03 08:27:51 -07002533TEST_F(VideoStreamEncoderTest,
2534 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002535 const int kWidth = 1280;
2536 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002537 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002538 DataRate::BitsPerSec(kTargetBitrateBps),
2539 DataRate::BitsPerSec(kTargetBitrateBps),
2540 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002541
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002542 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002543 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002544 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002545 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002546
2547 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002548 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002549 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002550 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002551 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2552
2553 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002554 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002555 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002556 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002557 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2558
mflodmancc3d4422017-08-03 08:27:51 -07002559 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002560}
2561
mflodmancc3d4422017-08-03 08:27:51 -07002562TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002563 const int kWidth = 1280;
2564 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002565 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002566 DataRate::BitsPerSec(kTargetBitrateBps),
2567 DataRate::BitsPerSec(kTargetBitrateBps),
2568 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002569
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002570 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002571 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002572 video_stream_encoder_->SetSource(&source,
2573 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002574
2575 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2576 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002577 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002578 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2579 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2580 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2581
2582 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002583 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002584 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002585 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2586 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2587 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2588
mflodmancc3d4422017-08-03 08:27:51 -07002589 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002590}
2591
mflodmancc3d4422017-08-03 08:27:51 -07002592TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002593 const int kWidth = 1280;
2594 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002595 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002596 DataRate::BitsPerSec(kTargetBitrateBps),
2597 DataRate::BitsPerSec(kTargetBitrateBps),
2598 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002599
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002600 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002601 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002602 video_stream_encoder_->SetSource(&source,
2603 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002604
2605 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2606 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002607 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002608 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2609 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2610 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2611
2612 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002613 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002614 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002615 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2616 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2617 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2618
mflodmancc3d4422017-08-03 08:27:51 -07002619 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002620}
2621
mflodmancc3d4422017-08-03 08:27:51 -07002622TEST_F(VideoStreamEncoderTest,
2623 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002624 const int kWidth = 1280;
2625 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002626 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002627 DataRate::BitsPerSec(kTargetBitrateBps),
2628 DataRate::BitsPerSec(kTargetBitrateBps),
2629 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002630
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002631 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002632 AdaptingFrameForwarder source;
2633 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002634 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002635 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002636
2637 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002638 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002639 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002640 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2641 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2642
2643 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002644 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002645 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002646 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002647 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2649 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2650
2651 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002652 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002653 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2655 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2656 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2657
mflodmancc3d4422017-08-03 08:27:51 -07002658 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002659}
2660
mflodmancc3d4422017-08-03 08:27:51 -07002661TEST_F(VideoStreamEncoderTest,
2662 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002663 const int kWidth = 1280;
2664 const int kHeight = 720;
2665 const int kInputFps = 30;
Erik Språng4c6ca302019-04-08 15:14:01 +02002666 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002667 DataRate::BitsPerSec(kTargetBitrateBps),
2668 DataRate::BitsPerSec(kTargetBitrateBps),
2669 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002670
2671 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2672 stats.input_frame_rate = kInputFps;
2673 stats_proxy_->SetMockStats(stats);
2674
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002675 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002676 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2677 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002678 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002679
2680 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002681 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002682 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2683 sink_.WaitForEncodedFrame(2);
2684 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2685
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002686 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002687 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002689 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002690 // Give the encoder queue time to process the change in degradation preference
2691 // by waiting for an encoded frame.
2692 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2693 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002694 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002695
2696 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002697 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002698 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2699 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002700 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2701
2702 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002703 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002704 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002705
mflodmancc3d4422017-08-03 08:27:51 -07002706 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002707}
2708
mflodmancc3d4422017-08-03 08:27:51 -07002709TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002710 const int kWidth = 1280;
2711 const int kHeight = 720;
2712 const size_t kNumFrames = 10;
2713
Erik Språng4c6ca302019-04-08 15:14:01 +02002714 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002715 DataRate::BitsPerSec(kTargetBitrateBps),
2716 DataRate::BitsPerSec(kTargetBitrateBps),
2717 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002718
asaperssond0de2952017-04-21 01:47:31 -07002719 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002720 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002721 video_source_.set_adaptation_enabled(true);
2722
2723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2725
2726 int downscales = 0;
2727 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002728 video_source_.IncomingCapturedFrame(
2729 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2730 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002731
asaperssonfab67072017-04-04 05:51:49 -07002732 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002733 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002734 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002735 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002736
2737 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2738 ++downscales;
2739
2740 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2741 EXPECT_EQ(downscales,
2742 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2743 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002744 }
mflodmancc3d4422017-08-03 08:27:51 -07002745 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002746}
2747
mflodmancc3d4422017-08-03 08:27:51 -07002748TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002749 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2750 const int kWidth = 1280;
2751 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002752 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002753 DataRate::BitsPerSec(kTargetBitrateBps),
2754 DataRate::BitsPerSec(kTargetBitrateBps),
2755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07002756
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002757 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07002758 AdaptingFrameForwarder source;
2759 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002760 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002761 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07002762
Åsa Persson8c1bf952018-09-13 10:42:19 +02002763 int64_t timestamp_ms = kFrameIntervalMs;
2764 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002765 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002766 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2768 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2769
2770 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002771 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002772 timestamp_ms += kFrameIntervalMs;
2773 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2774 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002775 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002776 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2777 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2778
2779 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002780 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002781 timestamp_ms += kFrameIntervalMs;
2782 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002783 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002784 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2786 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2787
2788 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002789 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002790 timestamp_ms += kFrameIntervalMs;
2791 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2792 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07002793 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07002794 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2795 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2796
2797 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002798 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002799 timestamp_ms += kFrameIntervalMs;
2800 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07002801 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002802 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07002803 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2804 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2805
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002807}
2808
mflodmancc3d4422017-08-03 08:27:51 -07002809TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07002810 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
2811 const int kWidth = 1280;
2812 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02002813 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002814 DataRate::BitsPerSec(kTargetBitrateBps),
2815 DataRate::BitsPerSec(kTargetBitrateBps),
2816 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002817
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002818 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002819 AdaptingFrameForwarder source;
2820 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002821 video_stream_encoder_->SetSource(&source,
2822 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002823
Åsa Persson8c1bf952018-09-13 10:42:19 +02002824 int64_t timestamp_ms = kFrameIntervalMs;
2825 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002826 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002827 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2830
2831 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002832 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002833 timestamp_ms += kFrameIntervalMs;
2834 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2835 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002836 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2837 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2838 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2839
2840 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002841 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002842 timestamp_ms += kFrameIntervalMs;
2843 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002844 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002845 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002846 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2847 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2848
2849 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002850 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002851 timestamp_ms += kFrameIntervalMs;
2852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
2853 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07002854 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2855 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2856 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2857
2858 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002860 timestamp_ms += kFrameIntervalMs;
2861 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07002862 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002863 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2865 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2866
mflodmancc3d4422017-08-03 08:27:51 -07002867 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002868}
2869
Sergey Silkin41c650b2019-10-14 13:12:19 +02002870TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
2871 fake_encoder_.SetResolutionBitrateLimits(
2872 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2873
2874 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002875 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2876 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2877 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2878 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002879
2880 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2881 AdaptingFrameForwarder source;
2882 source.set_adaptation_enabled(true);
2883 video_stream_encoder_->SetSource(
2884 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2885
2886 // Insert 720p frame.
2887 int64_t timestamp_ms = kFrameIntervalMs;
2888 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2889 WaitForEncodedFrame(1280, 720);
2890
2891 // Reduce bitrate and trigger adapt down.
2892 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002893 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2894 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2895 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2896 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002897 video_stream_encoder_->TriggerQualityLow();
2898
2899 // Insert 720p frame. It should be downscaled and encoded.
2900 timestamp_ms += kFrameIntervalMs;
2901 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2902 WaitForEncodedFrame(960, 540);
2903
2904 // Trigger adapt up. Higher resolution should not be requested duo to lack
2905 // of bitrate.
2906 video_stream_encoder_->TriggerQualityHigh();
2907 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2908
2909 // Increase bitrate.
2910 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002911 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2912 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
2913 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
2914 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002915
2916 // Trigger adapt up. Higher resolution should be requested.
2917 video_stream_encoder_->TriggerQualityHigh();
2918 VerifyFpsMaxResolutionMax(source.sink_wants());
2919
2920 video_stream_encoder_->Stop();
2921}
2922
2923TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
2924 fake_encoder_.SetResolutionBitrateLimits(
2925 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
2926
2927 // Set bitrate equal to min bitrate of 540p.
2928 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002929 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2930 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
2931 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
2932 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02002933
2934 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
2935 AdaptingFrameForwarder source;
2936 source.set_adaptation_enabled(true);
2937 video_stream_encoder_->SetSource(
2938 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2939
2940 // Insert 720p frame. It should be dropped and lower resolution should be
2941 // requested.
2942 int64_t timestamp_ms = kFrameIntervalMs;
2943 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2944 ExpectDroppedFrame();
2945 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
2946
2947 // Insert 720p frame. It should be downscaled and encoded.
2948 timestamp_ms += kFrameIntervalMs;
2949 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2950 WaitForEncodedFrame(960, 540);
2951
2952 video_stream_encoder_->Stop();
2953}
2954
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002955class BalancedDegradationTest : public VideoStreamEncoderTest {
2956 protected:
2957 void SetupTest() {
2958 // Reset encoder for field trials to take effect.
2959 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02002960 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002961
2962 // Enable BALANCED preference.
2963 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02002964 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
2965 }
2966
2967 void OnBitrateUpdated(int bitrate_bps) {
Ying Wang9b881ab2020-02-07 14:29:32 +01002968 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002969 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
2970 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002971 }
2972
Åsa Persson45b176f2019-09-30 11:19:05 +02002973 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002974 timestamp_ms_ += kFrameIntervalMs;
2975 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02002976 }
2977
2978 void InsertFrameAndWaitForEncoded() {
2979 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02002980 sink_.WaitForEncodedFrame(timestamp_ms_);
2981 }
2982
2983 const int kWidth = 640; // pixels:640x360=230400
2984 const int kHeight = 360;
2985 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
2986 int64_t timestamp_ms_ = 0;
2987 AdaptingFrameForwarder source_;
2988};
2989
2990TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
2991 test::ScopedFieldTrials field_trials(
2992 "WebRTC-Video-BalancedDegradationSettings/"
2993 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
2994 SetupTest();
2995
2996 // Force input frame rate.
2997 const int kInputFps = 24;
2998 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2999 stats.input_frame_rate = kInputFps;
3000 stats_proxy_->SetMockStats(stats);
3001
Åsa Persson45b176f2019-09-30 11:19:05 +02003002 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003003 VerifyFpsMaxResolutionMax(source_.sink_wants());
3004
3005 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
3006 // Fps diff (input-requested:0) < threshold, expect AdaptDown to return false.
3007 video_stream_encoder_->TriggerQualityLowExpectFalse();
3008 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3009
3010 video_stream_encoder_->Stop();
3011}
3012
3013TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3014 test::ScopedFieldTrials field_trials(
3015 "WebRTC-Video-BalancedDegradationSettings/"
3016 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3017 SetupTest();
3018
3019 // Force input frame rate.
3020 const int kInputFps = 25;
3021 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3022 stats.input_frame_rate = kInputFps;
3023 stats_proxy_->SetMockStats(stats);
3024
Åsa Persson45b176f2019-09-30 11:19:05 +02003025 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003026 VerifyFpsMaxResolutionMax(source_.sink_wants());
3027
3028 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
3029 // Fps diff (input-requested:1) == threshold, expect AdaptDown to return true.
3030 video_stream_encoder_->TriggerQualityLow();
3031 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3032
3033 video_stream_encoder_->Stop();
3034}
3035
3036TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3037 test::ScopedFieldTrials field_trials(
3038 "WebRTC-Video-BalancedDegradationSettings/"
3039 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3040 SetupTest();
3041
3042 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3043
Åsa Persson45b176f2019-09-30 11:19:05 +02003044 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003045 VerifyFpsMaxResolutionMax(source_.sink_wants());
3046
3047 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3048 video_stream_encoder_->TriggerQualityLow();
3049 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3050
3051 video_stream_encoder_->Stop();
3052}
3053
Åsa Perssonccfb3402019-09-25 15:13:04 +02003054TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003055 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003056 "WebRTC-Video-BalancedDegradationSettings/"
3057 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003058 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003059
Åsa Persson1b247f12019-08-14 17:26:39 +02003060 const int kMinBitrateBps = 425000;
3061 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003062 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003063
Åsa Persson45b176f2019-09-30 11:19:05 +02003064 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003065 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003066 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3067
3068 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3069 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003070 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003071 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003072 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3073
3074 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3075 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003076 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003077 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003078 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3079
Åsa Persson30ab0152019-08-27 12:22:33 +02003080 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3081 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003082 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003083 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3084 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003085 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3086
3087 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003088 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003089 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003090 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003091
Åsa Persson30ab0152019-08-27 12:22:33 +02003092 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003093 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003094 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003095 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003096 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003097 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3098
3099 video_stream_encoder_->Stop();
3100}
3101
Åsa Perssonccfb3402019-09-25 15:13:04 +02003102TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003103 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3104 test::ScopedFieldTrials field_trials(
3105 "WebRTC-Video-BalancedDegradationSettings/"
3106 "pixels:57600|129600|230400,fps:7|24|24/");
3107 SetupTest();
3108 OnBitrateUpdated(kLowTargetBitrateBps);
3109
3110 VerifyNoLimitation(source_.sink_wants());
3111
3112 // Insert frame, expect scaled down:
3113 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3114 InsertFrame();
3115 EXPECT_FALSE(WaitForFrame(1000));
3116 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3117 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3118
3119 // Insert frame, expect scaled down:
3120 // resolution (320x180@24fps).
3121 InsertFrame();
3122 EXPECT_FALSE(WaitForFrame(1000));
3123 EXPECT_LT(source_.sink_wants().max_pixel_count,
3124 source_.last_wants().max_pixel_count);
3125 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3126
3127 // Frame should not be dropped (min pixels per frame reached).
3128 InsertFrameAndWaitForEncoded();
3129
3130 video_stream_encoder_->Stop();
3131}
3132
3133TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003134 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003135 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003136 "WebRTC-Video-BalancedDegradationSettings/"
3137 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003138 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003139
Åsa Persson30ab0152019-08-27 12:22:33 +02003140 const int kResolutionMinBitrateBps = 435000;
3141 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003142 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003143
Åsa Persson45b176f2019-09-30 11:19:05 +02003144 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003145 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003146 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3147
3148 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3149 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003150 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003151 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003152 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3153
3154 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3155 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003156 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003157 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003158 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3159
3160 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3161 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003162 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003163 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003164 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3165
Åsa Persson30ab0152019-08-27 12:22:33 +02003166 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3167 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003168 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003169 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003170 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3171
3172 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3173 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003174 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003175 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3176
3177 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003178 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003179 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003180 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003181 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003182 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3183
3184 video_stream_encoder_->Stop();
3185}
3186
Åsa Perssonccfb3402019-09-25 15:13:04 +02003187TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003188 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003189 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003190 "WebRTC-Video-BalancedDegradationSettings/"
3191 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003192 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003193
Åsa Persson30ab0152019-08-27 12:22:33 +02003194 const int kMinBitrateBps = 425000;
3195 const int kTooLowMinBitrateBps = 424000;
3196 const int kResolutionMinBitrateBps = 435000;
3197 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003198 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003199
Åsa Persson45b176f2019-09-30 11:19:05 +02003200 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003201 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003202 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3203
3204 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3205 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003206 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003207 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003208 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3209
3210 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3211 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003212 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003213 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003214 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3215
3216 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3217 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003218 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003219 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003220 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3221
3222 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3223 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003224 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003225 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3226
3227 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003228 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003229 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003230 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003231 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003232 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3233
3234 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003235 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003236 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003237 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003238 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3239
3240 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003241 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003242 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003243 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003244 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003245 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3246
Åsa Persson1b247f12019-08-14 17:26:39 +02003247 video_stream_encoder_->Stop();
3248}
3249
mflodmancc3d4422017-08-03 08:27:51 -07003250TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003251 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3252 const int kWidth = 1280;
3253 const int kHeight = 720;
Erik Språng4c6ca302019-04-08 15:14:01 +02003254 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003255 DataRate::BitsPerSec(kTargetBitrateBps),
3256 DataRate::BitsPerSec(kTargetBitrateBps),
3257 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003258
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003259 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003260 AdaptingFrameForwarder source;
3261 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003262 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003263 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003264
Åsa Persson8c1bf952018-09-13 10:42:19 +02003265 int64_t timestamp_ms = kFrameIntervalMs;
3266 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003267 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003268 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003269 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3271 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3272 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3273
3274 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003275 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003276 timestamp_ms += kFrameIntervalMs;
3277 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3278 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003279 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003280 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3281 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3282 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3283 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3284
3285 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003286 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003287 timestamp_ms += kFrameIntervalMs;
3288 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3289 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003290 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003291 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3293 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3294 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3295
Jonathan Yubc771b72017-12-08 17:04:29 -08003296 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003297 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003298 timestamp_ms += kFrameIntervalMs;
3299 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3300 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003301 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003302 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3303 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003304 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003305 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3306
Jonathan Yubc771b72017-12-08 17:04:29 -08003307 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003308 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003309 timestamp_ms += kFrameIntervalMs;
3310 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3311 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003312 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003313 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003314 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3316 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3317 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3318
Jonathan Yubc771b72017-12-08 17:04:29 -08003319 // Trigger quality adapt down, expect no change (min resolution reached).
3320 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003321 timestamp_ms += kFrameIntervalMs;
3322 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3323 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003324 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3325 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3326 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3327 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3328 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3329
3330 // Trigger cpu adapt up, expect upscaled resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003331 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003332 timestamp_ms += kFrameIntervalMs;
3333 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3334 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003335 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003336 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3338 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3339 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3340
3341 // Trigger cpu adapt up, expect upscaled resolution (640x360).
3342 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003343 timestamp_ms += kFrameIntervalMs;
3344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3345 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003346 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3347 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3349 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3350 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351
3352 // Trigger cpu adapt up, expect upscaled resolution (960x540).
3353 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003354 timestamp_ms += kFrameIntervalMs;
3355 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3356 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003357 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003358 last_wants = source.sink_wants();
3359 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3360 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003361 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003362 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
3364 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003365 video_stream_encoder_->TriggerCpuNormalUsage();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003366 timestamp_ms += kFrameIntervalMs;
3367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3368 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003369 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003370 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003372 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003373 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3374
3375 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003376 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003377 timestamp_ms += kFrameIntervalMs;
3378 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003379 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003380 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003381 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003382 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3383 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003384 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003385 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003386
mflodmancc3d4422017-08-03 08:27:51 -07003387 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003388}
3389
mflodmancc3d4422017-08-03 08:27:51 -07003390TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003391 const int kWidth = 640;
3392 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003393
Erik Språng4c6ca302019-04-08 15:14:01 +02003394 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003395 DataRate::BitsPerSec(kTargetBitrateBps),
3396 DataRate::BitsPerSec(kTargetBitrateBps),
3397 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003398
perkj803d97f2016-11-01 11:45:46 -07003399 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003400 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003401 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003402 }
3403
mflodmancc3d4422017-08-03 08:27:51 -07003404 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003405 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003406 video_source_.IncomingCapturedFrame(CreateFrame(
3407 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003408 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003409 }
3410
mflodmancc3d4422017-08-03 08:27:51 -07003411 video_stream_encoder_->Stop();
3412 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003413 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003414
Ying Wangef3998f2019-12-09 13:06:53 +01003415 EXPECT_METRIC_EQ(
3416 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3417 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003418 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3419}
3420
mflodmancc3d4422017-08-03 08:27:51 -07003421TEST_F(VideoStreamEncoderTest,
3422 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Erik Språng4c6ca302019-04-08 15:14:01 +02003423 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003424 DataRate::BitsPerSec(kTargetBitrateBps),
3425 DataRate::BitsPerSec(kTargetBitrateBps),
3426 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003427 const int kWidth = 640;
3428 const int kHeight = 360;
3429
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003430 video_stream_encoder_->SetSource(&video_source_,
3431 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003432
3433 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3434 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003435 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003436 }
3437
mflodmancc3d4422017-08-03 08:27:51 -07003438 video_stream_encoder_->Stop();
3439 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003440 stats_proxy_.reset();
3441
3442 EXPECT_EQ(0,
3443 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3444}
3445
mflodmancc3d4422017-08-03 08:27:51 -07003446TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003447 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003448 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003449
3450 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003451 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003452 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003453 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3454 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003455
sprang57c2fff2017-01-16 06:24:02 -08003456 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003457 .Times(1);
Florent Castellia8336d32019-09-09 13:36:55 +02003458 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003459 DataRate::BitsPerSec(kLowTargetBitrateBps),
3460 DataRate::BitsPerSec(kLowTargetBitrateBps),
3461 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003462
sprang57c2fff2017-01-16 06:24:02 -08003463 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003464 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3465 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003466 VideoBitrateAllocation bitrate_allocation =
3467 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003468 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003469 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003470 // TODO(srte): The use of millisecs here looks like an error, but the tests
3471 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003472 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003473
3474 // Not called on second frame.
3475 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3476 .Times(0);
3477 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003478 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3479 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003480 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003481
3482 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003483 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3484 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003485 const int64_t start_time_ms = rtc::TimeMillis();
3486 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3487 video_source_.IncomingCapturedFrame(
3488 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3489 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003490 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003491 }
3492
3493 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003494 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003495
mflodmancc3d4422017-08-03 08:27:51 -07003496 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003497}
3498
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003499TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3500 // 2 TLs configured, temporal layers supported by encoder.
3501 const int kNumTemporalLayers = 2;
3502 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3503 fake_encoder_.SetTemporalLayersSupported(0, true);
3504
3505 // Bitrate allocated across temporal layers.
3506 const int kTl0Bps = kTargetBitrateBps *
3507 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003508 kNumTemporalLayers, /*temporal_id*/ 0,
3509 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003510 const int kTl1Bps = kTargetBitrateBps *
3511 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003512 kNumTemporalLayers, /*temporal_id*/ 1,
3513 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003514 VideoBitrateAllocation expected_bitrate;
3515 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3516 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3517
3518 VerifyAllocatedBitrate(expected_bitrate);
3519 video_stream_encoder_->Stop();
3520}
3521
3522TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3523 // 2 TLs configured, temporal layers not supported by encoder.
3524 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3525 fake_encoder_.SetTemporalLayersSupported(0, false);
3526
3527 // Temporal layers not supported by the encoder.
3528 // Total bitrate should be at ti:0.
3529 VideoBitrateAllocation expected_bitrate;
3530 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3531
3532 VerifyAllocatedBitrate(expected_bitrate);
3533 video_stream_encoder_->Stop();
3534}
3535
3536TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3537 // 2 TLs configured, temporal layers only supported for first stream.
3538 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3539 fake_encoder_.SetTemporalLayersSupported(0, true);
3540 fake_encoder_.SetTemporalLayersSupported(1, false);
3541
3542 const int kS0Bps = 150000;
3543 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003544 kS0Bps *
3545 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3546 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003547 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003548 kS0Bps *
3549 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3550 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003551 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3552 // Temporal layers not supported by si:1.
3553 VideoBitrateAllocation expected_bitrate;
3554 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3555 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3556 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3557
3558 VerifyAllocatedBitrate(expected_bitrate);
3559 video_stream_encoder_->Stop();
3560}
3561
Niels Möller7dc26b72017-12-06 10:27:48 +01003562TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3563 const int kFrameWidth = 1280;
3564 const int kFrameHeight = 720;
3565 const int kFramerate = 24;
3566
Erik Språng4c6ca302019-04-08 15:14:01 +02003567 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003568 DataRate::BitsPerSec(kTargetBitrateBps),
3569 DataRate::BitsPerSec(kTargetBitrateBps),
3570 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003571 test::FrameForwarder source;
3572 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003573 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003574
3575 // Insert a single frame, triggering initial configuration.
3576 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3577 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3578
3579 EXPECT_EQ(
3580 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3581 kDefaultFramerate);
3582
3583 // Trigger reconfigure encoder (without resetting the entire instance).
3584 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003585 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003586 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3587 video_encoder_config.number_of_streams = 1;
3588 video_encoder_config.video_stream_factory =
3589 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3590 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003591 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003592 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3593
3594 // Detector should be updated with fps limit from codec config.
3595 EXPECT_EQ(
3596 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3597 kFramerate);
3598
3599 // Trigger overuse, max framerate should be reduced.
3600 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3601 stats.input_frame_rate = kFramerate;
3602 stats_proxy_->SetMockStats(stats);
3603 video_stream_encoder_->TriggerCpuOveruse();
3604 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3605 int adapted_framerate =
3606 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3607 EXPECT_LT(adapted_framerate, kFramerate);
3608
3609 // Trigger underuse, max framerate should go back to codec configured fps.
3610 // Set extra low fps, to make sure it's actually reset, not just incremented.
3611 stats = stats_proxy_->GetStats();
3612 stats.input_frame_rate = adapted_framerate / 2;
3613 stats_proxy_->SetMockStats(stats);
3614 video_stream_encoder_->TriggerCpuNormalUsage();
3615 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3616 EXPECT_EQ(
3617 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3618 kFramerate);
3619
3620 video_stream_encoder_->Stop();
3621}
3622
3623TEST_F(VideoStreamEncoderTest,
3624 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3625 const int kFrameWidth = 1280;
3626 const int kFrameHeight = 720;
3627 const int kLowFramerate = 15;
3628 const int kHighFramerate = 25;
3629
Erik Språng4c6ca302019-04-08 15:14:01 +02003630 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003631 DataRate::BitsPerSec(kTargetBitrateBps),
3632 DataRate::BitsPerSec(kTargetBitrateBps),
3633 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003634 test::FrameForwarder source;
3635 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003636 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003637
3638 // Trigger initial configuration.
3639 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003640 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003641 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3642 video_encoder_config.number_of_streams = 1;
3643 video_encoder_config.video_stream_factory =
3644 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3645 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3646 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003647 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003648 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3649
3650 EXPECT_EQ(
3651 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3652 kLowFramerate);
3653
3654 // Trigger overuse, max framerate should be reduced.
3655 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3656 stats.input_frame_rate = kLowFramerate;
3657 stats_proxy_->SetMockStats(stats);
3658 video_stream_encoder_->TriggerCpuOveruse();
3659 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3660 int adapted_framerate =
3661 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3662 EXPECT_LT(adapted_framerate, kLowFramerate);
3663
3664 // Reconfigure the encoder with a new (higher max framerate), max fps should
3665 // still respect the adaptation.
3666 video_encoder_config.video_stream_factory =
3667 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3668 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3669 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003670 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003671 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3672
3673 EXPECT_EQ(
3674 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3675 adapted_framerate);
3676
3677 // Trigger underuse, max framerate should go back to codec configured fps.
3678 stats = stats_proxy_->GetStats();
3679 stats.input_frame_rate = adapted_framerate;
3680 stats_proxy_->SetMockStats(stats);
3681 video_stream_encoder_->TriggerCpuNormalUsage();
3682 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3683 EXPECT_EQ(
3684 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3685 kHighFramerate);
3686
3687 video_stream_encoder_->Stop();
3688}
3689
mflodmancc3d4422017-08-03 08:27:51 -07003690TEST_F(VideoStreamEncoderTest,
3691 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003692 const int kFrameWidth = 1280;
3693 const int kFrameHeight = 720;
3694 const int kFramerate = 24;
3695
Erik Språng4c6ca302019-04-08 15:14:01 +02003696 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003697 DataRate::BitsPerSec(kTargetBitrateBps),
3698 DataRate::BitsPerSec(kTargetBitrateBps),
3699 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003700 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003701 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003702 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003703
3704 // Trigger initial configuration.
3705 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003706 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003707 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3708 video_encoder_config.number_of_streams = 1;
3709 video_encoder_config.video_stream_factory =
3710 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3711 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003712 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003713 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003714 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003715
Niels Möller7dc26b72017-12-06 10:27:48 +01003716 EXPECT_EQ(
3717 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3718 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003719
3720 // Trigger overuse, max framerate should be reduced.
3721 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3722 stats.input_frame_rate = kFramerate;
3723 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003724 video_stream_encoder_->TriggerCpuOveruse();
3725 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003726 int adapted_framerate =
3727 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003728 EXPECT_LT(adapted_framerate, kFramerate);
3729
3730 // Change degradation preference to not enable framerate scaling. Target
3731 // framerate should be changed to codec defined limit.
mflodmancc3d4422017-08-03 08:27:51 -07003732 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003733 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -07003734 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003735 EXPECT_EQ(
3736 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3737 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003738
mflodmancc3d4422017-08-03 08:27:51 -07003739 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003740}
3741
mflodmancc3d4422017-08-03 08:27:51 -07003742TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003743 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003744 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003745 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3746 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3747 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003748 const int kWidth = 640;
3749 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003750
asaperssonfab67072017-04-04 05:51:49 -07003751 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003752
3753 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003754 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003755
3756 // Expect the sink_wants to specify a scaled frame.
asapersson0944a802017-04-07 00:57:58 -07003757 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003758
sprangc5d62e22017-04-02 23:53:04 -07003759 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08003760
asaperssonfab67072017-04-04 05:51:49 -07003761 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08003762 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003763 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08003764
3765 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07003766 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003767
sprangc5d62e22017-04-02 23:53:04 -07003768 EXPECT_LT(video_source_.sink_wants().max_pixel_count, last_pixel_count);
kthelgason2bc68642017-02-07 07:02:22 -08003769
mflodmancc3d4422017-08-03 08:27:51 -07003770 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003771}
3772
mflodmancc3d4422017-08-03 08:27:51 -07003773TEST_F(VideoStreamEncoderTest,
3774 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07003775 const int kTooLowBitrateForFrameSizeBps = 10000;
Erik Språng610c7632019-03-06 15:37:33 +01003776 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003777 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3778 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3779 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07003780 const int kWidth = 640;
3781 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08003782
3783 // We expect the n initial frames to get dropped.
3784 int i;
3785 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003786 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003787 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08003788 }
3789 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07003790 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003791 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08003792
3793 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07003794 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08003795
mflodmancc3d4422017-08-03 08:27:51 -07003796 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003797}
3798
mflodmancc3d4422017-08-03 08:27:51 -07003799TEST_F(VideoStreamEncoderTest,
3800 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07003801 const int kWidth = 640;
3802 const int kHeight = 360;
Florent Castellia8336d32019-09-09 13:36:55 +02003803 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003804 DataRate::BitsPerSec(kLowTargetBitrateBps),
3805 DataRate::BitsPerSec(kLowTargetBitrateBps),
3806 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08003807
3808 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07003809 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003810 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08003811
asaperssonfab67072017-04-04 05:51:49 -07003812 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08003813 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003814 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08003815
mflodmancc3d4422017-08-03 08:27:51 -07003816 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08003817}
3818
mflodmancc3d4422017-08-03 08:27:51 -07003819TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07003820 const int kWidth = 640;
3821 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08003822 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003823
3824 VideoEncoderConfig video_encoder_config;
3825 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3826 // Make format different, to force recreation of encoder.
3827 video_encoder_config.video_format.parameters["foo"] = "foo";
3828 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003829 kMaxPayloadLength);
Florent Castellia8336d32019-09-09 13:36:55 +02003830 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003831 DataRate::BitsPerSec(kLowTargetBitrateBps),
3832 DataRate::BitsPerSec(kLowTargetBitrateBps),
3833 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003834
kthelgasonb83797b2017-02-14 11:57:25 -08003835 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003836 video_stream_encoder_->SetSource(&video_source_,
3837 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08003838
asaperssonfab67072017-04-04 05:51:49 -07003839 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08003840 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07003841 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08003842
mflodmancc3d4422017-08-03 08:27:51 -07003843 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08003844 fake_encoder_.SetQualityScaling(true);
3845}
3846
Åsa Persson139f4dc2019-08-02 09:29:58 +02003847TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
3848 webrtc::test::ScopedFieldTrials field_trials(
3849 "WebRTC-Video-QualityScalerSettings/"
3850 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
3851 // Reset encoder for field trials to take effect.
3852 ConfigureEncoder(video_encoder_config_.Copy());
3853 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
3854 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
3855 const int kWidth = 640;
3856 const int kHeight = 360;
3857
3858 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003859 DataRate::BitsPerSec(kTargetBitrateBps),
3860 DataRate::BitsPerSec(kTargetBitrateBps),
3861 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003862 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3863 // Frame should not be dropped.
3864 WaitForEncodedFrame(1);
3865
3866 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003867 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3868 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
3869 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003870 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3871 // Frame should not be dropped.
3872 WaitForEncodedFrame(2);
3873
3874 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003875 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3876 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
3877 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02003878 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3879 // Expect to drop this frame, the wait should time out.
3880 ExpectDroppedFrame();
3881
3882 // Expect the sink_wants to specify a scaled frame.
3883 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
3884 video_stream_encoder_->Stop();
3885}
3886
Åsa Perssone644a032019-11-08 15:56:00 +01003887TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
3888 webrtc::test::ScopedFieldTrials field_trials(
3889 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
3890
3891 // Reset encoder for field trials to take effect.
3892 VideoEncoderConfig config = video_encoder_config_.Copy();
3893 config.max_bitrate_bps = kTargetBitrateBps;
3894 ConfigureEncoder(std::move(config));
3895 fake_encoder_.SetQp(kQpLow);
3896
3897 // Enable MAINTAIN_FRAMERATE preference.
3898 AdaptingFrameForwarder source;
3899 source.set_adaptation_enabled(true);
3900 video_stream_encoder_->SetSource(&source,
3901 DegradationPreference::MAINTAIN_FRAMERATE);
3902
3903 // Start at low bitrate.
3904 const int kLowBitrateBps = 200000;
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003905 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(kLowBitrateBps),
3906 DataRate::BitsPerSec(kLowBitrateBps),
3907 DataRate::BitsPerSec(kLowBitrateBps),
3908 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003909
3910 // Expect first frame to be dropped and resolution to be limited.
3911 const int kWidth = 1280;
3912 const int kHeight = 720;
3913 const int64_t kFrameIntervalMs = 100;
3914 int64_t timestamp_ms = kFrameIntervalMs;
3915 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3916 ExpectDroppedFrame();
3917 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3918
3919 // Increase bitrate to encoder max.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003920 video_stream_encoder_->OnBitrateUpdated(
3921 DataRate::BitsPerSec(config.max_bitrate_bps),
3922 DataRate::BitsPerSec(config.max_bitrate_bps),
3923 DataRate::BitsPerSec(config.max_bitrate_bps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01003924
3925 // Insert frames and advance |min_duration_ms|.
3926 for (size_t i = 1; i <= 10; i++) {
3927 timestamp_ms += kFrameIntervalMs;
3928 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3929 WaitForEncodedFrame(timestamp_ms);
3930 }
3931 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3932 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
3933
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003934 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01003935
3936 // Insert frame should trigger high BW and release quality limitation.
3937 timestamp_ms += kFrameIntervalMs;
3938 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3939 WaitForEncodedFrame(timestamp_ms);
3940 VerifyFpsMaxResolutionMax(source.sink_wants());
3941
3942 // Frame should not be adapted.
3943 timestamp_ms += kFrameIntervalMs;
3944 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3945 WaitForEncodedFrame(kWidth, kHeight);
3946 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3947
3948 video_stream_encoder_->Stop();
3949}
3950
mflodmancc3d4422017-08-03 08:27:51 -07003951TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003952 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
3953 const int kTooSmallWidth = 10;
3954 const int kTooSmallHeight = 10;
Erik Språng4c6ca302019-04-08 15:14:01 +02003955 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003956 DataRate::BitsPerSec(kTargetBitrateBps),
3957 DataRate::BitsPerSec(kTargetBitrateBps),
3958 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003959
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003960 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003961 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003962 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003963 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003964 VerifyNoLimitation(source.sink_wants());
3965 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3966
3967 // Trigger adapt down, too small frame, expect no change.
3968 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003969 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003970 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003971 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003972 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3973 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3974
mflodmancc3d4422017-08-03 08:27:51 -07003975 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003976}
3977
mflodmancc3d4422017-08-03 08:27:51 -07003978TEST_F(VideoStreamEncoderTest,
3979 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003980 const int kTooSmallWidth = 10;
3981 const int kTooSmallHeight = 10;
3982 const int kFpsLimit = 7;
Erik Språng4c6ca302019-04-08 15:14:01 +02003983 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003984 DataRate::BitsPerSec(kTargetBitrateBps),
3985 DataRate::BitsPerSec(kTargetBitrateBps),
3986 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003987
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003988 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003989 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003990 video_stream_encoder_->SetSource(&source,
3991 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003992 VerifyNoLimitation(source.sink_wants());
3993 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3994 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3995
3996 // Trigger adapt down, expect limited framerate.
3997 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07003998 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07003999 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004000 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4002 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4003 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4004
4005 // Trigger adapt down, too small frame, expect no change.
4006 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004007 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004008 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004009 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4010 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4011 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4012 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4013
mflodmancc3d4422017-08-03 08:27:51 -07004014 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004015}
4016
mflodmancc3d4422017-08-03 08:27:51 -07004017TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004018 fake_encoder_.ForceInitEncodeFailure(true);
Erik Språng4c6ca302019-04-08 15:14:01 +02004019 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004020 DataRate::BitsPerSec(kTargetBitrateBps),
4021 DataRate::BitsPerSec(kTargetBitrateBps),
4022 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004023 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004024 const int kFrameWidth = 1280;
4025 const int kFrameHeight = 720;
4026 video_source_.IncomingCapturedFrame(
4027 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004028 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004029 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004030}
4031
sprangb1ca0732017-02-01 08:38:12 -08004032// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004033TEST_F(VideoStreamEncoderTest,
4034 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004035 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004036 DataRate::BitsPerSec(kTargetBitrateBps),
4037 DataRate::BitsPerSec(kTargetBitrateBps),
4038 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004039
4040 const int kFrameWidth = 1280;
4041 const int kFrameHeight = 720;
4042 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004043 // requested by
4044 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004045 video_source_.set_adaptation_enabled(true);
4046
4047 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004048 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004049 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004050
4051 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004052 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004053 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004054 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004055 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004056
asaperssonfab67072017-04-04 05:51:49 -07004057 // Trigger CPU normal use, return to original resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004058 video_stream_encoder_->TriggerCpuNormalUsage();
sprangb1ca0732017-02-01 08:38:12 -08004059 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004060 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004061 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004062
mflodmancc3d4422017-08-03 08:27:51 -07004063 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004064}
sprangfe627f32017-03-29 08:24:59 -07004065
mflodmancc3d4422017-08-03 08:27:51 -07004066TEST_F(VideoStreamEncoderTest,
4067 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004068 const int kFrameWidth = 1280;
4069 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004070
Erik Språng4c6ca302019-04-08 15:14:01 +02004071 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004072 DataRate::BitsPerSec(kTargetBitrateBps),
4073 DataRate::BitsPerSec(kTargetBitrateBps),
4074 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004075 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004076 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004077 video_source_.set_adaptation_enabled(true);
4078
sprang4847ae62017-06-27 07:06:52 -07004079 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004080
4081 video_source_.IncomingCapturedFrame(
4082 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004083 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004084
4085 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004086 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004087
4088 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004089 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004090 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004091 video_source_.IncomingCapturedFrame(
4092 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004093 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004094 }
4095
4096 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004097 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004098 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004099 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004100 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004101 video_source_.IncomingCapturedFrame(
4102 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004103 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004104 ++num_frames_dropped;
4105 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004106 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004107 }
4108 }
4109
sprang4847ae62017-06-27 07:06:52 -07004110 // Add some slack to account for frames dropped by the frame dropper.
4111 const int kErrorMargin = 1;
4112 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004113 kErrorMargin);
4114
4115 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004116 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004117 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004118 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004119 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004120 video_source_.IncomingCapturedFrame(
4121 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004122 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004123 ++num_frames_dropped;
4124 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004125 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004126 }
4127 }
sprang4847ae62017-06-27 07:06:52 -07004128 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004129 kErrorMargin);
4130
4131 // Go back up one step.
mflodmancc3d4422017-08-03 08:27:51 -07004132 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07004133 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004134 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004135 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004136 video_source_.IncomingCapturedFrame(
4137 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004138 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004139 ++num_frames_dropped;
4140 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004141 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004142 }
4143 }
sprang4847ae62017-06-27 07:06:52 -07004144 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004145 kErrorMargin);
4146
4147 // Go back up to original mode.
mflodmancc3d4422017-08-03 08:27:51 -07004148 video_stream_encoder_->TriggerCpuNormalUsage();
sprangc5d62e22017-04-02 23:53:04 -07004149 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004150 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004151 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004152 video_source_.IncomingCapturedFrame(
4153 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004154 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004155 ++num_frames_dropped;
4156 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004157 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004158 }
4159 }
4160 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4161
mflodmancc3d4422017-08-03 08:27:51 -07004162 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004163}
4164
mflodmancc3d4422017-08-03 08:27:51 -07004165TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004166 const int kFramerateFps = 5;
4167 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004168 const int kFrameWidth = 1280;
4169 const int kFrameHeight = 720;
4170
sprang4847ae62017-06-27 07:06:52 -07004171 // Reconfigure encoder with two temporal layers and screensharing, which will
4172 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004173 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004174
Erik Språng4c6ca302019-04-08 15:14:01 +02004175 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004176 DataRate::BitsPerSec(kTargetBitrateBps),
4177 DataRate::BitsPerSec(kTargetBitrateBps),
4178 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004179 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004180 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004181 video_source_.set_adaptation_enabled(true);
4182
sprang4847ae62017-06-27 07:06:52 -07004183 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004184
4185 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004186 rtc::VideoSinkWants last_wants;
4187 do {
4188 last_wants = video_source_.sink_wants();
4189
sprangc5d62e22017-04-02 23:53:04 -07004190 // Insert frames to get a new fps estimate...
4191 for (int j = 0; j < kFramerateFps; ++j) {
4192 video_source_.IncomingCapturedFrame(
4193 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004194 if (video_source_.last_sent_width()) {
4195 sink_.WaitForEncodedFrame(timestamp_ms);
4196 }
sprangc5d62e22017-04-02 23:53:04 -07004197 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004198 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004199 }
4200 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004201 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004202 } while (video_source_.sink_wants().max_framerate_fps <
4203 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004204
Jonathan Yubc771b72017-12-08 17:04:29 -08004205 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004206
mflodmancc3d4422017-08-03 08:27:51 -07004207 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004208}
asaperssonf7e294d2017-06-13 23:25:22 -07004209
mflodmancc3d4422017-08-03 08:27:51 -07004210TEST_F(VideoStreamEncoderTest,
4211 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004212 const int kWidth = 1280;
4213 const int kHeight = 720;
4214 const int64_t kFrameIntervalMs = 150;
4215 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004216 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004217 DataRate::BitsPerSec(kTargetBitrateBps),
4218 DataRate::BitsPerSec(kTargetBitrateBps),
4219 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004220
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004221 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004222 AdaptingFrameForwarder source;
4223 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004224 video_stream_encoder_->SetSource(&source,
4225 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004226 timestamp_ms += kFrameIntervalMs;
4227 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004228 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004229 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004230 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4231 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4232 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4233
4234 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004235 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004236 timestamp_ms += kFrameIntervalMs;
4237 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004238 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004239 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4240 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4241 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4242 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4243
4244 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004245 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004246 timestamp_ms += kFrameIntervalMs;
4247 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004248 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004249 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4250 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4252 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4253
4254 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004255 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004256 timestamp_ms += kFrameIntervalMs;
4257 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004258 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004259 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4260 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4261 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4262 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4263
4264 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004265 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004266 timestamp_ms += kFrameIntervalMs;
4267 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004268 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004269 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4270 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4271 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4272 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4273
4274 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004275 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004276 timestamp_ms += kFrameIntervalMs;
4277 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004278 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004279 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4280 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4281 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4282 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4283
4284 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004285 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004286 timestamp_ms += kFrameIntervalMs;
4287 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004288 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004289 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4290 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4291 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4292 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4293
4294 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004295 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004296 timestamp_ms += kFrameIntervalMs;
4297 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004298 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004299 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4300 rtc::VideoSinkWants last_wants = source.sink_wants();
4301 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4302 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4303 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4304
4305 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004306 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004307 timestamp_ms += kFrameIntervalMs;
4308 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004309 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004310 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4311 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4312 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4313 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4314
4315 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004316 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004317 timestamp_ms += kFrameIntervalMs;
4318 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004319 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004320 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4321 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4322 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4323 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4324
4325 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004326 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004327 timestamp_ms += kFrameIntervalMs;
4328 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004329 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004330 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4331 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4332 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4333 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4334
4335 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004336 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004337 timestamp_ms += kFrameIntervalMs;
4338 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004339 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004340 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4341 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4342 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4343 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4344
4345 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004346 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004347 timestamp_ms += kFrameIntervalMs;
4348 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004349 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004350 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4351 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4352 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4353 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4354
4355 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004356 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004357 timestamp_ms += kFrameIntervalMs;
4358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004359 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004360 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4361 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4362 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4363 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4364
4365 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004366 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004367 timestamp_ms += kFrameIntervalMs;
4368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004369 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004370 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4372 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4373 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4374
Åsa Persson30ab0152019-08-27 12:22:33 +02004375 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004376 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004377 timestamp_ms += kFrameIntervalMs;
4378 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004379 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004380 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004381 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004382 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4383 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4384 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4385
4386 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004387 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004388 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004389 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
mflodmancc3d4422017-08-03 08:27:51 -07004391 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004392}
4393
mflodmancc3d4422017-08-03 08:27:51 -07004394TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004395 const int kWidth = 1280;
4396 const int kHeight = 720;
4397 const int64_t kFrameIntervalMs = 150;
4398 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004399 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004400 DataRate::BitsPerSec(kTargetBitrateBps),
4401 DataRate::BitsPerSec(kTargetBitrateBps),
4402 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004403
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004404 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004405 AdaptingFrameForwarder source;
4406 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004407 video_stream_encoder_->SetSource(&source,
4408 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004409 timestamp_ms += kFrameIntervalMs;
4410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004411 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004412 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4415 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4416 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4417 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4418 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4419
4420 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004421 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004422 timestamp_ms += kFrameIntervalMs;
4423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004424 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004425 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4428 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4429 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4430 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4431 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4432
4433 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004434 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004435 timestamp_ms += kFrameIntervalMs;
4436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004437 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004438 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4439 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4440 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4441 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4443 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4445
4446 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004447 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004448 timestamp_ms += kFrameIntervalMs;
4449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004450 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004451 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4454 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4455 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4456 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4457 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4458
4459 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004460 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004461 timestamp_ms += kFrameIntervalMs;
4462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004463 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004464 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4465 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4467 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4468 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4469 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4470 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4471
4472 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004473 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004474 timestamp_ms += kFrameIntervalMs;
4475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004476 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004477 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4481 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4482 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4483 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4484
4485 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004486 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004487 timestamp_ms += kFrameIntervalMs;
4488 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004489 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004490 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004491 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004492 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4493 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4494 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4495 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4496 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4497 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4498
4499 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004500 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004501 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004502 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4503 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4504
mflodmancc3d4422017-08-03 08:27:51 -07004505 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004506}
4507
mflodmancc3d4422017-08-03 08:27:51 -07004508TEST_F(VideoStreamEncoderTest,
4509 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004510 const int kWidth = 640;
4511 const int kHeight = 360;
4512 const int kFpsLimit = 15;
4513 const int64_t kFrameIntervalMs = 150;
4514 int64_t timestamp_ms = kFrameIntervalMs;
Erik Språng4c6ca302019-04-08 15:14:01 +02004515 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004516 DataRate::BitsPerSec(kTargetBitrateBps),
4517 DataRate::BitsPerSec(kTargetBitrateBps),
4518 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004519
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004520 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004521 AdaptingFrameForwarder source;
4522 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004523 video_stream_encoder_->SetSource(&source,
4524 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004525 timestamp_ms += kFrameIntervalMs;
4526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004527 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004528 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4533 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4534 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4535
4536 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004537 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004538 timestamp_ms += kFrameIntervalMs;
4539 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004540 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004541 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4542 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4543 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4544 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4545 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4546 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4547 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4548
4549 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004550 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004551 timestamp_ms += kFrameIntervalMs;
4552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004553 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004554 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4556 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4557 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4558 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4559 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4560 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4561
4562 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004563 video_stream_encoder_->TriggerCpuNormalUsage();
asaperssonf7e294d2017-06-13 23:25:22 -07004564 timestamp_ms += kFrameIntervalMs;
4565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004566 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004567 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4568 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4570 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4571 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4572 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4573 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4574
4575 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004576 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004577 timestamp_ms += kFrameIntervalMs;
4578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004579 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004580 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4582 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4583 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4584 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4585 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4586 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4587
4588 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004589 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004590 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004591 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4592 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4593
mflodmancc3d4422017-08-03 08:27:51 -07004594 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004595}
4596
mflodmancc3d4422017-08-03 08:27:51 -07004597TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004598 const int kFrameWidth = 1920;
4599 const int kFrameHeight = 1080;
4600 // 3/4 of 1920.
4601 const int kAdaptedFrameWidth = 1440;
4602 // 3/4 of 1080 rounded down to multiple of 4.
4603 const int kAdaptedFrameHeight = 808;
4604 const int kFramerate = 24;
4605
Erik Språng4c6ca302019-04-08 15:14:01 +02004606 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004607 DataRate::BitsPerSec(kTargetBitrateBps),
4608 DataRate::BitsPerSec(kTargetBitrateBps),
4609 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004610 // Trigger reconfigure encoder (without resetting the entire instance).
4611 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004612 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004613 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4614 video_encoder_config.number_of_streams = 1;
4615 video_encoder_config.video_stream_factory =
4616 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004617 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004618 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004619 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004620
4621 video_source_.set_adaptation_enabled(true);
4622
4623 video_source_.IncomingCapturedFrame(
4624 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004625 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004626
4627 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004628 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004629 video_source_.IncomingCapturedFrame(
4630 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004631 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004632
mflodmancc3d4422017-08-03 08:27:51 -07004633 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004634}
4635
mflodmancc3d4422017-08-03 08:27:51 -07004636TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004637 const int kFrameWidth = 1280;
4638 const int kFrameHeight = 720;
4639 const int kLowFps = 2;
4640 const int kHighFps = 30;
4641
Erik Språng4c6ca302019-04-08 15:14:01 +02004642 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004643 DataRate::BitsPerSec(kTargetBitrateBps),
4644 DataRate::BitsPerSec(kTargetBitrateBps),
4645 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004646
4647 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4648 max_framerate_ = kLowFps;
4649
4650 // Insert 2 seconds of 2fps video.
4651 for (int i = 0; i < kLowFps * 2; ++i) {
4652 video_source_.IncomingCapturedFrame(
4653 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4654 WaitForEncodedFrame(timestamp_ms);
4655 timestamp_ms += 1000 / kLowFps;
4656 }
4657
4658 // Make sure encoder is updated with new target.
Erik Språng4c6ca302019-04-08 15:14:01 +02004659 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004660 DataRate::BitsPerSec(kTargetBitrateBps),
4661 DataRate::BitsPerSec(kTargetBitrateBps),
4662 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004663 video_source_.IncomingCapturedFrame(
4664 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4665 WaitForEncodedFrame(timestamp_ms);
4666 timestamp_ms += 1000 / kLowFps;
4667
4668 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4669
4670 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004671 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004672 const int kFrameIntervalMs = 1000 / kHighFps;
4673 max_framerate_ = kHighFps;
4674 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4675 video_source_.IncomingCapturedFrame(
4676 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4677 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4678 // be dropped if the encoder hans't been updated with the new higher target
4679 // framerate yet, causing it to overshoot the target bitrate and then
4680 // suffering the wrath of the media optimizer.
4681 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4682 timestamp_ms += kFrameIntervalMs;
4683 }
4684
4685 // Don expect correct measurement just yet, but it should be higher than
4686 // before.
4687 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4688
mflodmancc3d4422017-08-03 08:27:51 -07004689 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004690}
4691
mflodmancc3d4422017-08-03 08:27:51 -07004692TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004693 const int kFrameWidth = 1280;
4694 const int kFrameHeight = 720;
4695 const int kTargetBitrateBps = 1000000;
4696
4697 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004698 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Erik Språng4c6ca302019-04-08 15:14:01 +02004699 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004700 DataRate::BitsPerSec(kTargetBitrateBps),
4701 DataRate::BitsPerSec(kTargetBitrateBps),
4702 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004703 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004704
4705 // Insert a first video frame, causes another bitrate update.
4706 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4707 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4708 video_source_.IncomingCapturedFrame(
4709 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4710 WaitForEncodedFrame(timestamp_ms);
4711
4712 // Next, simulate video suspension due to pacer queue overrun.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004713 video_stream_encoder_->OnBitrateUpdated(DataRate::BitsPerSec(0),
4714 DataRate::BitsPerSec(0),
4715 DataRate::BitsPerSec(0), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004716
4717 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004718 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004719 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004720
4721 // Bitrate observer should not be called.
4722 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4723 video_source_.IncomingCapturedFrame(
4724 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4725 ExpectDroppedFrame();
4726
mflodmancc3d4422017-08-03 08:27:51 -07004727 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004728}
ilnik6b826ef2017-06-16 06:53:48 -07004729
Niels Möller4db138e2018-04-19 09:04:13 +02004730TEST_F(VideoStreamEncoderTest,
4731 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
4732 const int kFrameWidth = 1280;
4733 const int kFrameHeight = 720;
4734 const CpuOveruseOptions default_options;
Erik Språng4c6ca302019-04-08 15:14:01 +02004735 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004736 DataRate::BitsPerSec(kTargetBitrateBps),
4737 DataRate::BitsPerSec(kTargetBitrateBps),
4738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004739 video_source_.IncomingCapturedFrame(
4740 CreateFrame(1, kFrameWidth, kFrameHeight));
4741 WaitForEncodedFrame(1);
4742 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4743 .low_encode_usage_threshold_percent,
4744 default_options.low_encode_usage_threshold_percent);
4745 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4746 .high_encode_usage_threshold_percent,
4747 default_options.high_encode_usage_threshold_percent);
4748 video_stream_encoder_->Stop();
4749}
4750
4751TEST_F(VideoStreamEncoderTest,
4752 HigherCpuAdaptationThresholdsForHardwareEncoder) {
4753 const int kFrameWidth = 1280;
4754 const int kFrameHeight = 720;
4755 CpuOveruseOptions hardware_options;
4756 hardware_options.low_encode_usage_threshold_percent = 150;
4757 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01004758 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02004759
Erik Språng4c6ca302019-04-08 15:14:01 +02004760 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004761 DataRate::BitsPerSec(kTargetBitrateBps),
4762 DataRate::BitsPerSec(kTargetBitrateBps),
4763 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02004764 video_source_.IncomingCapturedFrame(
4765 CreateFrame(1, kFrameWidth, kFrameHeight));
4766 WaitForEncodedFrame(1);
4767 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4768 .low_encode_usage_threshold_percent,
4769 hardware_options.low_encode_usage_threshold_percent);
4770 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
4771 .high_encode_usage_threshold_percent,
4772 hardware_options.high_encode_usage_threshold_percent);
4773 video_stream_encoder_->Stop();
4774}
4775
Niels Möller6bb5ab92019-01-11 11:11:10 +01004776TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
4777 const int kFrameWidth = 320;
4778 const int kFrameHeight = 240;
4779 const int kFps = 30;
4780 const int kTargetBitrateBps = 120000;
4781 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
4782
Erik Språng4c6ca302019-04-08 15:14:01 +02004783 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004784 DataRate::BitsPerSec(kTargetBitrateBps),
4785 DataRate::BitsPerSec(kTargetBitrateBps),
4786 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004787
4788 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4789 max_framerate_ = kFps;
4790
4791 // Insert 3 seconds of video, verify number of drops with normal bitrate.
4792 fake_encoder_.SimulateOvershoot(1.0);
4793 int num_dropped = 0;
4794 for (int i = 0; i < kNumFramesInRun; ++i) {
4795 video_source_.IncomingCapturedFrame(
4796 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4797 // Wait up to two frame durations for a frame to arrive.
4798 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4799 ++num_dropped;
4800 }
4801 timestamp_ms += 1000 / kFps;
4802 }
4803
Erik Språnga8d48ab2019-02-08 14:17:40 +01004804 // Framerate should be measured to be near the expected target rate.
4805 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4806
4807 // Frame drops should be within 5% of expected 0%.
4808 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004809
4810 // Make encoder produce frames at double the expected bitrate during 3 seconds
4811 // of video, verify number of drops. Rate needs to be slightly changed in
4812 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01004813 double overshoot_factor = 2.0;
4814 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
4815 // With bitrate adjuster, when need to overshoot even more to trigger
4816 // frame dropping.
4817 overshoot_factor *= 2;
4818 }
4819 fake_encoder_.SimulateOvershoot(overshoot_factor);
Erik Språng610c7632019-03-06 15:37:33 +01004820 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004821 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4822 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
4823 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004824 num_dropped = 0;
4825 for (int i = 0; i < kNumFramesInRun; ++i) {
4826 video_source_.IncomingCapturedFrame(
4827 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4828 // Wait up to two frame durations for a frame to arrive.
4829 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
4830 ++num_dropped;
4831 }
4832 timestamp_ms += 1000 / kFps;
4833 }
4834
Erik Språng4c6ca302019-04-08 15:14:01 +02004835 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004836 DataRate::BitsPerSec(kTargetBitrateBps),
4837 DataRate::BitsPerSec(kTargetBitrateBps),
4838 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01004839
4840 // Target framerate should be still be near the expected target, despite
4841 // the frame drops.
4842 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
4843
4844 // Frame drops should be within 5% of expected 50%.
4845 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004846
4847 video_stream_encoder_->Stop();
4848}
4849
4850TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
4851 const int kFrameWidth = 320;
4852 const int kFrameHeight = 240;
4853 const int kActualInputFps = 24;
4854 const int kTargetBitrateBps = 120000;
4855
4856 ASSERT_GT(max_framerate_, kActualInputFps);
4857
4858 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4859 max_framerate_ = kActualInputFps;
Erik Språng4c6ca302019-04-08 15:14:01 +02004860 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004861 DataRate::BitsPerSec(kTargetBitrateBps),
4862 DataRate::BitsPerSec(kTargetBitrateBps),
4863 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01004864
4865 // Insert 3 seconds of video, with an input fps lower than configured max.
4866 for (int i = 0; i < kActualInputFps * 3; ++i) {
4867 video_source_.IncomingCapturedFrame(
4868 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4869 // Wait up to two frame durations for a frame to arrive.
4870 WaitForEncodedFrame(timestamp_ms);
4871 timestamp_ms += 1000 / kActualInputFps;
4872 }
4873
4874 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
4875
4876 video_stream_encoder_->Stop();
4877}
4878
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004879TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
4880 VideoFrame::UpdateRect rect;
Erik Språng4c6ca302019-04-08 15:14:01 +02004881 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004882 DataRate::BitsPerSec(kTargetBitrateBps),
4883 DataRate::BitsPerSec(kTargetBitrateBps),
4884 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01004885
4886 fake_encoder_.BlockNextEncode();
4887 video_source_.IncomingCapturedFrame(
4888 CreateFrameWithUpdatedPixel(1, nullptr, 0));
4889 WaitForEncodedFrame(1);
4890 // On the very first frame full update should be forced.
4891 rect = fake_encoder_.GetLastUpdateRect();
4892 EXPECT_EQ(rect.offset_x, 0);
4893 EXPECT_EQ(rect.offset_y, 0);
4894 EXPECT_EQ(rect.height, codec_height_);
4895 EXPECT_EQ(rect.width, codec_width_);
4896 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
4897 // call to ContinueEncode.
4898 video_source_.IncomingCapturedFrame(
4899 CreateFrameWithUpdatedPixel(2, nullptr, 1));
4900 ExpectDroppedFrame();
4901 video_source_.IncomingCapturedFrame(
4902 CreateFrameWithUpdatedPixel(3, nullptr, 10));
4903 ExpectDroppedFrame();
4904 fake_encoder_.ContinueEncode();
4905 WaitForEncodedFrame(3);
4906 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
4907 rect = fake_encoder_.GetLastUpdateRect();
4908 EXPECT_EQ(rect.offset_x, 1);
4909 EXPECT_EQ(rect.offset_y, 0);
4910 EXPECT_EQ(rect.width, 10);
4911 EXPECT_EQ(rect.height, 1);
4912
4913 video_source_.IncomingCapturedFrame(
4914 CreateFrameWithUpdatedPixel(4, nullptr, 0));
4915 WaitForEncodedFrame(4);
4916 // Previous frame was encoded, so no accumulation should happen.
4917 rect = fake_encoder_.GetLastUpdateRect();
4918 EXPECT_EQ(rect.offset_x, 0);
4919 EXPECT_EQ(rect.offset_y, 0);
4920 EXPECT_EQ(rect.width, 1);
4921 EXPECT_EQ(rect.height, 1);
4922
4923 video_stream_encoder_->Stop();
4924}
4925
Erik Språngd7329ca2019-02-21 21:19:53 +01004926TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Erik Språng4c6ca302019-04-08 15:14:01 +02004927 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004928 DataRate::BitsPerSec(kTargetBitrateBps),
4929 DataRate::BitsPerSec(kTargetBitrateBps),
4930 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004931
4932 // First frame is always keyframe.
4933 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4934 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01004935 EXPECT_THAT(
4936 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004937 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004938
4939 // Insert delta frame.
4940 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4941 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01004942 EXPECT_THAT(
4943 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004944 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004945
4946 // Request next frame be a key-frame.
4947 video_stream_encoder_->SendKeyFrame();
4948 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4949 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01004950 EXPECT_THAT(
4951 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004952 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004953
4954 video_stream_encoder_->Stop();
4955}
4956
4957TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
4958 // Setup simulcast with three streams.
4959 ResetEncoder("VP8", 3, 1, 1, false);
Erik Språng610c7632019-03-06 15:37:33 +01004960 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004961 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4962 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4963 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01004964 // Wait for all three layers before triggering event.
4965 sink_.SetNumExpectedLayers(3);
4966
4967 // First frame is always keyframe.
4968 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
4969 WaitForEncodedFrame(1);
4970 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004971 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
4972 VideoFrameType::kVideoFrameKey,
4973 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004974
4975 // Insert delta frame.
4976 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
4977 WaitForEncodedFrame(2);
4978 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004979 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
4980 VideoFrameType::kVideoFrameDelta,
4981 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004982
4983 // Request next frame be a key-frame.
4984 // Only first stream is configured to produce key-frame.
4985 video_stream_encoder_->SendKeyFrame();
4986 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
4987 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02004988
4989 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
4990 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01004991 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02004992 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02004993 VideoFrameType::kVideoFrameKey,
4994 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01004995
4996 video_stream_encoder_->Stop();
4997}
4998
4999TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5000 // Configure internal source factory and setup test again.
5001 encoder_factory_.SetHasInternalSource(true);
5002 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005003 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005004 DataRate::BitsPerSec(kTargetBitrateBps),
5005 DataRate::BitsPerSec(kTargetBitrateBps),
5006 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005007
5008 // Call encoder directly, simulating internal source where encoded frame
5009 // callback in VideoStreamEncoder is called despite no OnFrame().
5010 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5011 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005012 EXPECT_THAT(
5013 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005014 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005015
Niels Möller8f7ce222019-03-21 15:43:58 +01005016 const std::vector<VideoFrameType> kDeltaFrame = {
5017 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005018 // Need to set timestamp manually since manually for injected frame.
5019 VideoFrame frame = CreateFrame(101, nullptr);
5020 frame.set_timestamp(101);
5021 fake_encoder_.InjectFrame(frame, false);
5022 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005023 EXPECT_THAT(
5024 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005025 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005026
5027 // Request key-frame. The forces a dummy frame down into the encoder.
5028 fake_encoder_.ExpectNullFrame();
5029 video_stream_encoder_->SendKeyFrame();
5030 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005031 EXPECT_THAT(
5032 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005033 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005034
5035 video_stream_encoder_->Stop();
5036}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005037
5038TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5039 // Configure internal source factory and setup test again.
5040 encoder_factory_.SetHasInternalSource(true);
5041 ResetEncoder("VP8", 1, 1, 1, false);
Erik Språng4c6ca302019-04-08 15:14:01 +02005042 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005043 DataRate::BitsPerSec(kTargetBitrateBps),
5044 DataRate::BitsPerSec(kTargetBitrateBps),
5045 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005046
5047 int64_t timestamp = 1;
5048 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005049 image.SetEncodedData(
5050 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005051 image.capture_time_ms_ = ++timestamp;
5052 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5053 const int64_t kEncodeFinishDelayMs = 10;
5054 image.timing_.encode_start_ms = timestamp;
5055 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5056 fake_encoder_.InjectEncodedImage(image);
5057 // Wait for frame without incrementing clock.
5058 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5059 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5060 // capture timestamp should be kEncodeFinishDelayMs in the past.
5061 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5062 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5063 kEncodeFinishDelayMs);
5064
5065 video_stream_encoder_->Stop();
5066}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005067
5068TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5069 // Configure internal source factory and setup test again.
5070 encoder_factory_.SetHasInternalSource(true);
5071 ResetEncoder("H264", 1, 1, 1, false);
5072
5073 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5074 image._frameType = VideoFrameType::kVideoFrameKey;
5075
5076 CodecSpecificInfo codec_specific_info;
5077 codec_specific_info.codecType = kVideoCodecH264;
5078
5079 RTPFragmentationHeader fragmentation;
5080 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5081 fragmentation.fragmentationOffset[0] = 4;
5082 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5083
5084 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5085 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5086
5087 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5088 testing::ElementsAreArray(optimal_sps));
5089 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5090 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5091 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5092 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5093
5094 video_stream_encoder_->Stop();
5095}
5096
5097TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5098 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5099 0x00, 0x00, 0x03, 0x03, 0xF4,
5100 0x05, 0x03, 0xC7, 0xC0};
5101
5102 // Configure internal source factory and setup test again.
5103 encoder_factory_.SetHasInternalSource(true);
5104 ResetEncoder("H264", 1, 1, 1, false);
5105
5106 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5107 image._frameType = VideoFrameType::kVideoFrameKey;
5108
5109 CodecSpecificInfo codec_specific_info;
5110 codec_specific_info.codecType = kVideoCodecH264;
5111
5112 RTPFragmentationHeader fragmentation;
5113 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5114 fragmentation.fragmentationOffset[0] = 4;
5115 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5116
5117 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5118 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5119
5120 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5121 testing::ElementsAreArray(optimal_sps));
5122 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5123 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5124 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5125 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5126
5127 video_stream_encoder_->Stop();
5128}
5129
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005130TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5131 const int kFrameWidth = 1280;
5132 const int kFrameHeight = 720;
5133 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5134
5135 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005136 DataRate::BitsPerSec(kTargetBitrateBps),
5137 DataRate::BitsPerSec(kTargetBitrateBps),
5138 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005139 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5140
5141 // Insert a first video frame. It should be dropped because of downscale in
5142 // resolution.
5143 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5144 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5145 frame.set_rotation(kVideoRotation_270);
5146 video_source_.IncomingCapturedFrame(frame);
5147
5148 ExpectDroppedFrame();
5149
5150 // Second frame is downscaled.
5151 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5152 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5153 frame.set_rotation(kVideoRotation_90);
5154 video_source_.IncomingCapturedFrame(frame);
5155
5156 WaitForEncodedFrame(timestamp_ms);
5157 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5158
5159 // Insert another frame, also downscaled.
5160 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5161 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5162 frame.set_rotation(kVideoRotation_180);
5163 video_source_.IncomingCapturedFrame(frame);
5164
5165 WaitForEncodedFrame(timestamp_ms);
5166 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5167
5168 video_stream_encoder_->Stop();
5169}
5170
Erik Språng5056af02019-09-02 15:53:11 +02005171TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5172 const int kFrameWidth = 320;
5173 const int kFrameHeight = 180;
5174
5175 // Initial rate.
5176 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005177 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5178 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5179 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005180 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005181 /*rtt_ms=*/0,
5182 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005183
5184 // Insert a first video frame so that encoder gets configured.
5185 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5186 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5187 frame.set_rotation(kVideoRotation_270);
5188 video_source_.IncomingCapturedFrame(frame);
5189 WaitForEncodedFrame(timestamp_ms);
5190
5191 // Set a target rate below the minimum allowed by the codec settings.
5192 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005193 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5194 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Erik Språng5056af02019-09-02 15:53:11 +02005195 video_stream_encoder_->OnBitrateUpdated(
5196 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005197 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005198 /*link_allocation=*/target_rate,
5199 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005200 /*rtt_ms=*/0,
5201 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005202 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5203
5204 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5205 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5206 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005207 DataRate allocation_sum =
5208 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005209 EXPECT_EQ(min_rate, allocation_sum);
5210 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5211
5212 video_stream_encoder_->Stop();
5213}
5214
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005215TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005216 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005217 DataRate::BitsPerSec(kTargetBitrateBps),
5218 DataRate::BitsPerSec(kTargetBitrateBps),
5219 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005220 // Capture a frame and wait for it to synchronize with the encoder thread.
5221 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5222 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5223 WaitForEncodedFrame(1);
5224
5225 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5226 ASSERT_TRUE(prev_rate_settings.has_value());
5227 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5228 kDefaultFramerate);
5229
5230 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5231 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5232 timestamp_ms += 1000 / kDefaultFramerate;
5233 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5234 WaitForEncodedFrame(timestamp_ms);
5235 }
5236 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5237 kDefaultFramerate);
5238 // Capture larger frame to trigger a reconfigure.
5239 codec_height_ *= 2;
5240 codec_width_ *= 2;
5241 timestamp_ms += 1000 / kDefaultFramerate;
5242 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5243 WaitForEncodedFrame(timestamp_ms);
5244
5245 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5246 auto current_rate_settings =
5247 fake_encoder_.GetAndResetLastRateControlSettings();
5248 // Ensure we have actually reconfigured twice
5249 // The rate settings should have been set again even though
5250 // they haven't changed.
5251 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005252 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005253
5254 video_stream_encoder_->Stop();
5255}
5256
philipeld9cc8c02019-09-16 14:53:40 +02005257struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
5258 MOCK_METHOD0(RequestEncoderFallback, void());
5259 MOCK_METHOD1(RequestEncoderSwitch, void(const Config& conf));
philipel9b058032020-02-10 11:30:00 +01005260 MOCK_METHOD1(RequestEncoderSwitch,
5261 void(const webrtc::SdpVideoFormat& format));
philipeld9cc8c02019-09-16 14:53:40 +02005262};
5263
5264TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5265 constexpr int kDontCare = 100;
5266
5267 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5268 video_send_config_.encoder_settings.encoder_switch_request_callback =
5269 &switch_callback;
5270 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5271 encoder_config.codec_type = kVideoCodecVP8;
5272 webrtc::test::ScopedFieldTrials field_trial(
5273 "WebRTC-NetworkCondition-EncoderSwitch/"
5274 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5275 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5276
5277 // Reset encoder for new configuration to take effect.
5278 ConfigureEncoder(std::move(encoder_config));
5279
5280 // Send one frame to trigger ReconfigureEncoder.
5281 video_source_.IncomingCapturedFrame(
5282 CreateFrame(kDontCare, kDontCare, kDontCare));
5283
5284 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005285 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5286 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005287 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005288 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005289
5290 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005291 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5292 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5293 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005294 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005295 /*rtt_ms=*/0,
5296 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005297
5298 video_stream_encoder_->Stop();
5299}
5300
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005301TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5302 constexpr int kDontCare = 100;
5303
5304 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5305 video_send_config_.encoder_settings.encoder_switch_request_callback =
5306 &switch_callback;
5307 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5308 encoder_config.codec_type = kVideoCodecVP8;
5309 webrtc::test::ScopedFieldTrials field_trial(
5310 "WebRTC-NetworkCondition-EncoderSwitch/"
5311 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5312 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5313
5314 // Reset encoder for new configuration to take effect.
5315 ConfigureEncoder(std::move(encoder_config));
5316
5317 // Send one frame to trigger ReconfigureEncoder.
5318 video_source_.IncomingCapturedFrame(
5319 CreateFrame(kDontCare, kDontCare, kDontCare));
5320
5321 using Config = EncoderSwitchRequestCallback::Config;
5322 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5323 .Times(0);
5324
5325 video_stream_encoder_->OnBitrateUpdated(
5326 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5327 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5328 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5329 /*fraction_lost=*/0,
5330 /*rtt_ms=*/0,
5331 /*cwnd_reduce_ratio=*/0);
5332
5333 video_stream_encoder_->Stop();
5334}
5335
philipeld9cc8c02019-09-16 14:53:40 +02005336TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5337 constexpr int kSufficientBitrateToNotDrop = 1000;
5338 constexpr int kHighRes = 500;
5339 constexpr int kLowRes = 100;
5340
5341 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5342 video_send_config_.encoder_settings.encoder_switch_request_callback =
5343 &switch_callback;
5344 webrtc::test::ScopedFieldTrials field_trial(
5345 "WebRTC-NetworkCondition-EncoderSwitch/"
5346 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5347 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5348 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5349 encoder_config.codec_type = kVideoCodecH264;
5350
5351 // Reset encoder for new configuration to take effect.
5352 ConfigureEncoder(std::move(encoder_config));
5353
5354 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5355 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5356 // not fail.
5357 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005358 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5359 /*stable_target_bitrate=*/
5360 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5361 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005362 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005363 /*rtt_ms=*/0,
5364 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005365
5366 // Send one frame to trigger ReconfigureEncoder.
5367 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5368 WaitForEncodedFrame(1);
5369
5370 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005371 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5372 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005373 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005374 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005375
5376 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5377 WaitForEncodedFrame(2);
5378
5379 video_stream_encoder_->Stop();
5380}
5381
philipel9b058032020-02-10 11:30:00 +01005382TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5383 constexpr int kDontCare = 100;
5384 StrictMock<MockEncoderSelector> encoder_selector;
5385 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5386 &fake_encoder_, &encoder_selector);
5387 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5388
5389 // Reset encoder for new configuration to take effect.
5390 ConfigureEncoder(video_encoder_config_.Copy());
5391
5392 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5393
5394 video_source_.IncomingCapturedFrame(
5395 CreateFrame(kDontCare, kDontCare, kDontCare));
5396 video_stream_encoder_->Stop();
5397
5398 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5399 // to it's factory, so in order for the encoder instance in the
5400 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5401 // reset the |video_stream_encoder_| here.
5402 video_stream_encoder_.reset();
5403}
5404
5405TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5406 constexpr int kDontCare = 100;
5407
5408 NiceMock<MockEncoderSelector> encoder_selector;
5409 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5410 video_send_config_.encoder_settings.encoder_switch_request_callback =
5411 &switch_callback;
5412 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5413 &fake_encoder_, &encoder_selector);
5414 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5415
5416 // Reset encoder for new configuration to take effect.
5417 ConfigureEncoder(video_encoder_config_.Copy());
5418
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005419 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005420 .WillByDefault(Return(SdpVideoFormat("AV1")));
5421 EXPECT_CALL(switch_callback,
5422 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5423 Field(&SdpVideoFormat::name, "AV1"))));
5424
5425 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005426 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5427 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5428 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005429 /*fraction_lost=*/0,
5430 /*rtt_ms=*/0,
5431 /*cwnd_reduce_ratio=*/0);
5432
5433 video_stream_encoder_->Stop();
5434}
5435
5436TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5437 constexpr int kSufficientBitrateToNotDrop = 1000;
5438 constexpr int kDontCare = 100;
5439
5440 NiceMock<MockVideoEncoder> video_encoder;
5441 NiceMock<MockEncoderSelector> encoder_selector;
5442 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5443 video_send_config_.encoder_settings.encoder_switch_request_callback =
5444 &switch_callback;
5445 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5446 &video_encoder, &encoder_selector);
5447 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5448
5449 // Reset encoder for new configuration to take effect.
5450 ConfigureEncoder(video_encoder_config_.Copy());
5451
5452 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5453 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5454 // not fail.
5455 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005456 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5457 /*stable_target_bitrate=*/
5458 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5459 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005460 /*fraction_lost=*/0,
5461 /*rtt_ms=*/0,
5462 /*cwnd_reduce_ratio=*/0);
5463
5464 ON_CALL(video_encoder, Encode(_, _))
5465 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5466 ON_CALL(encoder_selector, OnEncoderBroken())
5467 .WillByDefault(Return(SdpVideoFormat("AV2")));
5468
5469 rtc::Event encode_attempted;
5470 EXPECT_CALL(switch_callback,
5471 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5472 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5473 EXPECT_EQ(format.name, "AV2");
5474 encode_attempted.Set();
5475 });
5476
5477 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5478 encode_attempted.Wait(3000);
5479
5480 video_stream_encoder_->Stop();
5481
5482 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5483 // to it's factory, so in order for the encoder instance in the
5484 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5485 // reset the |video_stream_encoder_| here.
5486 video_stream_encoder_.reset();
5487}
5488
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005489TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005490 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005491 const int kFrameWidth = 320;
5492 const int kFrameHeight = 180;
5493
5494 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005495 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005496 video_stream_encoder_->OnBitrateUpdated(
5497 /*target_bitrate=*/rate,
5498 /*stable_target_bitrate=*/rate,
5499 /*link_allocation=*/rate,
5500 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005501 /*rtt_ms=*/0,
5502 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005503
5504 // Insert a first video frame so that encoder gets configured.
5505 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5506 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5507 frame.set_rotation(kVideoRotation_270);
5508 video_source_.IncomingCapturedFrame(frame);
5509 WaitForEncodedFrame(timestamp_ms);
5510 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5511
5512 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005513 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005514 video_stream_encoder_->OnBitrateUpdated(
5515 /*target_bitrate=*/new_stable_rate,
5516 /*stable_target_bitrate=*/new_stable_rate,
5517 /*link_allocation=*/rate,
5518 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005519 /*rtt_ms=*/0,
5520 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005521 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5522 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5523 video_stream_encoder_->Stop();
5524}
5525
5526TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005527 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005528 const int kFrameWidth = 320;
5529 const int kFrameHeight = 180;
5530
5531 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005532 auto rate = DataRate::KilobitsPerSec(100);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005533 video_stream_encoder_->OnBitrateUpdated(
5534 /*target_bitrate=*/rate,
5535 /*stable_target_bitrate=*/rate,
5536 /*link_allocation=*/rate,
5537 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005538 /*rtt_ms=*/0,
5539 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005540
5541 // Insert a first video frame so that encoder gets configured.
5542 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5543 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5544 frame.set_rotation(kVideoRotation_270);
5545 video_source_.IncomingCapturedFrame(frame);
5546 WaitForEncodedFrame(timestamp_ms);
5547 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5548
5549 // Set a higher target rate without changing the link_allocation. Should not
5550 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005551 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005552 video_stream_encoder_->OnBitrateUpdated(
5553 /*target_bitrate=*/rate,
5554 /*stable_target_bitrate=*/new_stable_rate,
5555 /*link_allocation=*/rate,
5556 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005557 /*rtt_ms=*/0,
5558 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005559 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5560 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5561 video_stream_encoder_->Stop();
5562}
5563
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005564TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5565 test::ScopedFieldTrials field_trials(
5566 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5567 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5568 const int kFramerateFps = 30;
5569 const int kWidth = 1920;
5570 const int kHeight = 1080;
5571 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5572 // Works on screenshare mode.
5573 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5574 // We rely on the automatic resolution adaptation, but we handle framerate
5575 // adaptation manually by mocking the stats proxy.
5576 video_source_.set_adaptation_enabled(true);
5577
5578 // BALANCED degradation preference is required for this feature.
5579 video_stream_encoder_->OnBitrateUpdated(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005580 DataRate::BitsPerSec(kTargetBitrateBps),
5581 DataRate::BitsPerSec(kTargetBitrateBps),
5582 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005583 video_stream_encoder_->SetSource(&video_source_,
5584 webrtc::DegradationPreference::BALANCED);
5585 VerifyNoLimitation(video_source_.sink_wants());
5586
5587 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5588 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5589
5590 // Pass enough frames with the full update to trigger animation detection.
5591 for (int i = 0; i < kNumFrames; ++i) {
5592 int64_t timestamp_ms =
5593 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5594 frame.set_ntp_time_ms(timestamp_ms);
5595 frame.set_timestamp_us(timestamp_ms * 1000);
5596 video_source_.IncomingCapturedFrame(frame);
5597 WaitForEncodedFrame(timestamp_ms);
5598 }
5599
5600 // Resolution should be limited.
5601 rtc::VideoSinkWants expected;
5602 expected.max_framerate_fps = kFramerateFps;
5603 expected.max_pixel_count = 1280 * 720 + 1;
5604 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5605
5606 // Pass one frame with no known update.
5607 // Resolution cap should be removed immediately.
5608 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5609 frame.set_ntp_time_ms(timestamp_ms);
5610 frame.set_timestamp_us(timestamp_ms * 1000);
5611 frame.clear_update_rect();
5612
5613 video_source_.IncomingCapturedFrame(frame);
5614 WaitForEncodedFrame(timestamp_ms);
5615
5616 // Resolution should be unlimited now.
5617 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5618
5619 video_stream_encoder_->Stop();
5620}
5621
perkj26091b12016-09-01 01:17:40 -07005622} // namespace webrtc