blob: 9292933d8947a206b757e6913cc9e0bc556fb384 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020037#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080039#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010040#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020041#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "system_wrappers/include/sleep.h"
43#include "test/encoder_settings.h"
44#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020045#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010046#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020047#include "test/gmock.h"
48#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020049#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070051
52namespace webrtc {
53
sprang57c2fff2017-01-16 06:24:02 -080054using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020055using ::testing::AllOf;
56using ::testing::Field;
philipel9b058032020-02-10 11:30:00 +010057using ::testing::Matcher;
58using ::testing::NiceMock;
59using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020060using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080061
perkj803d97f2016-11-01 11:45:46 -070062namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020063const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010064const int kQpLow = 1;
65const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020066const int kMinFramerateFps = 2;
67const int kMinBalancedFramerateFps = 7;
68const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080069const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010070const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020071const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010072const uint32_t kSimulcastTargetBitrateBps = 3150000;
73const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080074const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070075const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020076const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020077const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020078const VideoEncoder::ResolutionBitrateLimits
79 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
80const VideoEncoder::ResolutionBitrateLimits
81 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080082
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020083uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
84 0x00, 0x00, 0x03, 0x03, 0xF4,
85 0x05, 0x03, 0xC7, 0xE0, 0x1B,
86 0x41, 0x10, 0x8D, 0x00};
87
perkj803d97f2016-11-01 11:45:46 -070088class TestBuffer : public webrtc::I420Buffer {
89 public:
90 TestBuffer(rtc::Event* event, int width, int height)
91 : I420Buffer(width, height), event_(event) {}
92
93 private:
94 friend class rtc::RefCountedObject<TestBuffer>;
95 ~TestBuffer() override {
96 if (event_)
97 event_->Set();
98 }
99 rtc::Event* const event_;
100};
101
Noah Richards51db4212019-06-12 06:59:12 -0700102// A fake native buffer that can't be converted to I420.
103class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
104 public:
105 FakeNativeBuffer(rtc::Event* event, int width, int height)
106 : event_(event), width_(width), height_(height) {}
107 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
108 int width() const override { return width_; }
109 int height() const override { return height_; }
110 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
111 return nullptr;
112 }
113
114 private:
115 friend class rtc::RefCountedObject<FakeNativeBuffer>;
116 ~FakeNativeBuffer() override {
117 if (event_)
118 event_->Set();
119 }
120 rtc::Event* const event_;
121 const int width_;
122 const int height_;
123};
124
Niels Möller7dc26b72017-12-06 10:27:48 +0100125class CpuOveruseDetectorProxy : public OveruseFrameDetector {
126 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200127 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
128 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200129 last_target_framerate_fps_(-1),
130 framerate_updated_event_(true /* manual_reset */,
131 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100132 virtual ~CpuOveruseDetectorProxy() {}
133
134 void OnTargetFramerateUpdated(int framerate_fps) override {
135 rtc::CritScope cs(&lock_);
136 last_target_framerate_fps_ = framerate_fps;
137 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200138 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100139 }
140
141 int GetLastTargetFramerate() {
142 rtc::CritScope cs(&lock_);
143 return last_target_framerate_fps_;
144 }
145
Niels Möller4db138e2018-04-19 09:04:13 +0200146 CpuOveruseOptions GetOptions() { return options_; }
147
Henrik Boström381d1092020-05-12 18:49:07 +0200148 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
149
Niels Möller7dc26b72017-12-06 10:27:48 +0100150 private:
151 rtc::CriticalSection lock_;
152 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200153 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100154};
155
Henrik Boström91aa7322020-04-28 12:24:33 +0200156class FakeQualityScalerQpUsageHandlerCallback
157 : public QualityScalerQpUsageHandlerCallbackInterface {
158 public:
159 FakeQualityScalerQpUsageHandlerCallback()
Henrik Boström5bf60e42020-05-13 15:20:25 +0200160 : QualityScalerQpUsageHandlerCallbackInterface(),
161 qp_usage_handled_event_(/*manual_reset=*/true,
162 /*initially_signaled=*/false),
163 clear_qp_samples_result_(absl::nullopt) {}
164 ~FakeQualityScalerQpUsageHandlerCallback() override {
165 RTC_DCHECK(clear_qp_samples_result_.has_value());
166 }
Henrik Boström91aa7322020-04-28 12:24:33 +0200167
168 void OnQpUsageHandled(bool clear_qp_samples) override {
169 clear_qp_samples_result_ = clear_qp_samples;
Henrik Boström5bf60e42020-05-13 15:20:25 +0200170 qp_usage_handled_event_.Set();
Henrik Boström91aa7322020-04-28 12:24:33 +0200171 }
172
Henrik Boström5bf60e42020-05-13 15:20:25 +0200173 bool WaitForQpUsageHandled() { return qp_usage_handled_event_.Wait(5000); }
174
Henrik Boström91aa7322020-04-28 12:24:33 +0200175 absl::optional<bool> clear_qp_samples_result() const {
176 return clear_qp_samples_result_;
177 }
178
179 private:
Henrik Boström5bf60e42020-05-13 15:20:25 +0200180 rtc::Event qp_usage_handled_event_;
Henrik Boström91aa7322020-04-28 12:24:33 +0200181 absl::optional<bool> clear_qp_samples_result_;
182};
183
Henrik Boström381d1092020-05-12 18:49:07 +0200184class VideoSourceRestrictionsUpdatedListener
185 : public ResourceAdaptationProcessorListener {
186 public:
187 VideoSourceRestrictionsUpdatedListener()
188 : was_restrictions_updated_(false), restrictions_updated_event_() {}
189 ~VideoSourceRestrictionsUpdatedListener() override {
190 RTC_DCHECK(was_restrictions_updated_);
191 }
192
193 rtc::Event* restrictions_updated_event() {
194 return &restrictions_updated_event_;
195 }
196
197 // ResourceAdaptationProcessorListener implementation.
198 void OnVideoSourceRestrictionsUpdated(
199 VideoSourceRestrictions restrictions,
200 const VideoAdaptationCounters& adaptation_counters,
201 rtc::scoped_refptr<Resource> reason) override {
202 was_restrictions_updated_ = true;
203 restrictions_updated_event_.Set();
204 }
205
206 private:
207 bool was_restrictions_updated_;
208 rtc::Event restrictions_updated_event_;
209};
210
mflodmancc3d4422017-08-03 08:27:51 -0700211class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700212 public:
Niels Möller213618e2018-07-24 09:29:58 +0200213 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200214 const VideoStreamEncoderSettings& settings,
215 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100216 : VideoStreamEncoder(Clock::GetRealTimeClock(),
217 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200218 stats_proxy,
219 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200220 std::unique_ptr<OveruseFrameDetector>(
221 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100222 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100223 task_queue_factory),
Henrik Boströmc55516d2020-05-11 16:29:22 +0200224 fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
225 fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
Henrik Boström381d1092020-05-12 18:49:07 +0200226 fake_cpu_resource_->Initialize(encoder_queue(),
227 resource_adaptation_queue());
228 fake_quality_resource_->Initialize(encoder_queue(),
229 resource_adaptation_queue());
Henrik Boströmc55516d2020-05-11 16:29:22 +0200230 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200231 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200232 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100233 }
perkj803d97f2016-11-01 11:45:46 -0700234
Henrik Boström381d1092020-05-12 18:49:07 +0200235 void SetSourceAndWaitForRestrictionsUpdated(
236 rtc::VideoSourceInterface<VideoFrame>* source,
237 const DegradationPreference& degradation_preference) {
238 VideoSourceRestrictionsUpdatedListener listener;
239 AddAdaptationListenerForTesting(&listener);
240 SetSource(source, degradation_preference);
241 listener.restrictions_updated_event()->Wait(5000);
242 RemoveAdaptationListenerForTesting(&listener);
243 }
244
245 void SetSourceAndWaitForFramerateUpdated(
246 rtc::VideoSourceInterface<VideoFrame>* source,
247 const DegradationPreference& degradation_preference) {
248 overuse_detector_proxy_->framerate_updated_event()->Reset();
249 SetSource(source, degradation_preference);
250 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
251 }
252
253 void OnBitrateUpdatedAndWaitForManagedResources(
254 DataRate target_bitrate,
255 DataRate stable_target_bitrate,
256 DataRate link_allocation,
257 uint8_t fraction_lost,
258 int64_t round_trip_time_ms,
259 double cwnd_reduce_ratio) {
260 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
261 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
262 // Bitrate is updated on the encoder queue.
263 WaitUntilTaskQueueIsIdle();
264 // Give the managed resources time to react to the new bitrate.
265 // TODO(hbos): Can we await an appropriate event instead?
266 WaitUntilAdaptationTaskQueueIsIdle();
267 }
268
269 void WaitUntilAdaptationTaskQueueIsIdle() {
270 rtc::Event event;
271 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
272 ASSERT_TRUE(event.Wait(5000));
273 }
274
kthelgason2fc52542017-03-03 00:24:41 -0800275 // This is used as a synchronisation mechanism, to make sure that the
276 // encoder queue is not blocked before we start sending it frames.
277 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100278 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200279 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800280 ASSERT_TRUE(event.Wait(5000));
281 }
282
Henrik Boström91aa7322020-04-28 12:24:33 +0200283 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200284 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200285 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200286 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200287 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
288 event.Set();
289 });
290 ASSERT_TRUE(event.Wait(5000));
291 }
292 void TriggerCpuUnderuse() {
293 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200294 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200295 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
296 event.Set();
297 });
298 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200299 }
kthelgason876222f2016-11-29 01:44:11 -0800300
Henrik Boström91aa7322020-04-28 12:24:33 +0200301 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200302 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200303 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200304 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200305 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
306 event.Set();
307 });
308 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200309 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200310 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200311 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200312 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200313 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
314 event.Set();
315 });
316 ASSERT_TRUE(event.Wait(5000));
317 }
318
319 // Fakes high QP resource usage measurements on the real
320 // QualityScalerResource. Returns whether or not QP samples would have been
321 // cleared if this had been a real signal from the QualityScaler.
322 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200323 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
324 new FakeQualityScalerQpUsageHandlerCallback();
Henrik Boström5bf60e42020-05-13 15:20:25 +0200325 encoder_queue()->PostTask([this, callback] {
326 // This will cause a "ping" between adaptation task queue and encoder
327 // queue. When we have the result, the |callback| will be notified.
Henrik Boström91aa7322020-04-28 12:24:33 +0200328 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
Henrik Boström91aa7322020-04-28 12:24:33 +0200329 });
Henrik Boström5bf60e42020-05-13 15:20:25 +0200330 EXPECT_TRUE(callback->WaitForQpUsageHandled());
Henrik Boström91aa7322020-04-28 12:24:33 +0200331 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
332 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200333 }
sprangfda496a2017-06-15 04:21:07 -0700334
Niels Möller7dc26b72017-12-06 10:27:48 +0100335 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200336 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
337 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700338};
339
asapersson5f7226f2016-11-25 04:37:00 -0800340class VideoStreamFactory
341 : public VideoEncoderConfig::VideoStreamFactoryInterface {
342 public:
sprangfda496a2017-06-15 04:21:07 -0700343 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
344 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800345 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700346 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800347 }
348
349 private:
350 std::vector<VideoStream> CreateEncoderStreams(
351 int width,
352 int height,
353 const VideoEncoderConfig& encoder_config) override {
354 std::vector<VideoStream> streams =
355 test::CreateVideoStreams(width, height, encoder_config);
356 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100357 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700358 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800359 }
360 return streams;
361 }
sprangfda496a2017-06-15 04:21:07 -0700362
asapersson5f7226f2016-11-25 04:37:00 -0800363 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700364 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800365};
366
Noah Richards51db4212019-06-12 06:59:12 -0700367// Simulates simulcast behavior and makes highest stream resolutions divisible
368// by 4.
369class CroppingVideoStreamFactory
370 : public VideoEncoderConfig::VideoStreamFactoryInterface {
371 public:
372 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
373 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
374 EXPECT_GT(num_temporal_layers, 0u);
375 EXPECT_GT(framerate, 0);
376 }
377
378 private:
379 std::vector<VideoStream> CreateEncoderStreams(
380 int width,
381 int height,
382 const VideoEncoderConfig& encoder_config) override {
383 std::vector<VideoStream> streams = test::CreateVideoStreams(
384 width - width % 4, height - height % 4, encoder_config);
385 for (VideoStream& stream : streams) {
386 stream.num_temporal_layers = num_temporal_layers_;
387 stream.max_framerate = framerate_;
388 }
389 return streams;
390 }
391
392 const size_t num_temporal_layers_;
393 const int framerate_;
394};
395
sprangb1ca0732017-02-01 08:38:12 -0800396class AdaptingFrameForwarder : public test::FrameForwarder {
397 public:
398 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700399 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800400
401 void set_adaptation_enabled(bool enabled) {
402 rtc::CritScope cs(&crit_);
403 adaptation_enabled_ = enabled;
404 }
405
asaperssonfab67072017-04-04 05:51:49 -0700406 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800407 rtc::CritScope cs(&crit_);
408 return adaptation_enabled_;
409 }
410
asapersson09f05612017-05-15 23:40:18 -0700411 rtc::VideoSinkWants last_wants() const {
412 rtc::CritScope cs(&crit_);
413 return last_wants_;
414 }
415
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200416 absl::optional<int> last_sent_width() const { return last_width_; }
417 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800418
sprangb1ca0732017-02-01 08:38:12 -0800419 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
420 int cropped_width = 0;
421 int cropped_height = 0;
422 int out_width = 0;
423 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700424 if (adaption_enabled()) {
425 if (adapter_.AdaptFrameResolution(
426 video_frame.width(), video_frame.height(),
427 video_frame.timestamp_us() * 1000, &cropped_width,
428 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100429 VideoFrame adapted_frame =
430 VideoFrame::Builder()
431 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
432 nullptr, out_width, out_height))
433 .set_timestamp_rtp(99)
434 .set_timestamp_ms(99)
435 .set_rotation(kVideoRotation_0)
436 .build();
sprangc5d62e22017-04-02 23:53:04 -0700437 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100438 if (video_frame.has_update_rect()) {
439 adapted_frame.set_update_rect(
440 video_frame.update_rect().ScaleWithFrame(
441 video_frame.width(), video_frame.height(), 0, 0,
442 video_frame.width(), video_frame.height(), out_width,
443 out_height));
444 }
sprangc5d62e22017-04-02 23:53:04 -0700445 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800446 last_width_.emplace(adapted_frame.width());
447 last_height_.emplace(adapted_frame.height());
448 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200449 last_width_ = absl::nullopt;
450 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700451 }
sprangb1ca0732017-02-01 08:38:12 -0800452 } else {
453 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800454 last_width_.emplace(video_frame.width());
455 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800456 }
457 }
458
459 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
460 const rtc::VideoSinkWants& wants) override {
461 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700462 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100463 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800464 test::FrameForwarder::AddOrUpdateSink(sink, wants);
465 }
sprangb1ca0732017-02-01 08:38:12 -0800466 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700467 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
468 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200469 absl::optional<int> last_width_;
470 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800471};
sprangc5d62e22017-04-02 23:53:04 -0700472
Niels Möller213618e2018-07-24 09:29:58 +0200473// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700474class MockableSendStatisticsProxy : public SendStatisticsProxy {
475 public:
476 MockableSendStatisticsProxy(Clock* clock,
477 const VideoSendStream::Config& config,
478 VideoEncoderConfig::ContentType content_type)
479 : SendStatisticsProxy(clock, config, content_type) {}
480
481 VideoSendStream::Stats GetStats() override {
482 rtc::CritScope cs(&lock_);
483 if (mock_stats_)
484 return *mock_stats_;
485 return SendStatisticsProxy::GetStats();
486 }
487
Niels Möller213618e2018-07-24 09:29:58 +0200488 int GetInputFrameRate() const override {
489 rtc::CritScope cs(&lock_);
490 if (mock_stats_)
491 return mock_stats_->input_frame_rate;
492 return SendStatisticsProxy::GetInputFrameRate();
493 }
sprangc5d62e22017-04-02 23:53:04 -0700494 void SetMockStats(const VideoSendStream::Stats& stats) {
495 rtc::CritScope cs(&lock_);
496 mock_stats_.emplace(stats);
497 }
498
499 void ResetMockStats() {
500 rtc::CritScope cs(&lock_);
501 mock_stats_.reset();
502 }
503
504 private:
505 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200506 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700507};
508
sprang4847ae62017-06-27 07:06:52 -0700509class MockBitrateObserver : public VideoBitrateAllocationObserver {
510 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200511 MOCK_METHOD(void,
512 OnBitrateAllocationUpdated,
513 (const VideoBitrateAllocation&),
514 (override));
sprang4847ae62017-06-27 07:06:52 -0700515};
516
philipel9b058032020-02-10 11:30:00 +0100517class MockEncoderSelector
518 : public VideoEncoderFactory::EncoderSelectorInterface {
519 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200520 MOCK_METHOD(void,
521 OnCurrentEncoder,
522 (const SdpVideoFormat& format),
523 (override));
524 MOCK_METHOD(absl::optional<SdpVideoFormat>,
525 OnAvailableBitrate,
526 (const DataRate& rate),
527 (override));
528 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100529};
530
perkj803d97f2016-11-01 11:45:46 -0700531} // namespace
532
mflodmancc3d4422017-08-03 08:27:51 -0700533class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700534 public:
535 static const int kDefaultTimeoutMs = 30 * 1000;
536
mflodmancc3d4422017-08-03 08:27:51 -0700537 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700538 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700539 codec_width_(320),
540 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200541 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200542 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700543 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200544 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700545 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700546 Clock::GetRealTimeClock(),
547 video_send_config_,
548 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700549 sink_(&fake_encoder_) {}
550
551 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700552 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700553 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200554 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800555 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200556 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200557 video_send_config_.rtp.payload_name = "FAKE";
558 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700559
Per512ecb32016-09-23 15:52:06 +0200560 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200561 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700562 video_encoder_config.video_stream_factory =
563 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100564 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700565
566 // Framerate limit is specified by the VideoStreamFactory.
567 std::vector<VideoStream> streams =
568 video_encoder_config.video_stream_factory->CreateEncoderStreams(
569 codec_width_, codec_height_, video_encoder_config);
570 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100571 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700572
Niels Möllerf1338562018-04-26 09:51:47 +0200573 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800574 }
575
Niels Möllerf1338562018-04-26 09:51:47 +0200576 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700577 if (video_stream_encoder_)
578 video_stream_encoder_->Stop();
579 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200580 stats_proxy_.get(), video_send_config_.encoder_settings,
581 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700582 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
583 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700584 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700585 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
586 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200587 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700588 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800589 }
590
591 void ResetEncoder(const std::string& payload_name,
592 size_t num_streams,
593 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700594 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700595 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200596 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800597
598 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200599 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800600 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100601 video_encoder_config.max_bitrate_bps =
602 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800603 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700604 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
605 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700606 video_encoder_config.content_type =
607 screenshare ? VideoEncoderConfig::ContentType::kScreen
608 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700609 if (payload_name == "VP9") {
610 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
611 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
612 video_encoder_config.encoder_specific_settings =
613 new rtc::RefCountedObject<
614 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
615 }
Niels Möllerf1338562018-04-26 09:51:47 +0200616 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700617 }
618
sprang57c2fff2017-01-16 06:24:02 -0800619 VideoFrame CreateFrame(int64_t ntp_time_ms,
620 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100621 VideoFrame frame =
622 VideoFrame::Builder()
623 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
624 destruction_event, codec_width_, codec_height_))
625 .set_timestamp_rtp(99)
626 .set_timestamp_ms(99)
627 .set_rotation(kVideoRotation_0)
628 .build();
sprang57c2fff2017-01-16 06:24:02 -0800629 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700630 return frame;
631 }
632
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100633 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
634 rtc::Event* destruction_event,
635 int offset_x) const {
636 VideoFrame frame =
637 VideoFrame::Builder()
638 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
639 destruction_event, codec_width_, codec_height_))
640 .set_timestamp_rtp(99)
641 .set_timestamp_ms(99)
642 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100643 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100644 .build();
645 frame.set_ntp_time_ms(ntp_time_ms);
646 return frame;
647 }
648
sprang57c2fff2017-01-16 06:24:02 -0800649 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100650 VideoFrame frame =
651 VideoFrame::Builder()
652 .set_video_frame_buffer(
653 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
654 .set_timestamp_rtp(99)
655 .set_timestamp_ms(99)
656 .set_rotation(kVideoRotation_0)
657 .build();
sprang57c2fff2017-01-16 06:24:02 -0800658 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700659 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700660 return frame;
661 }
662
Noah Richards51db4212019-06-12 06:59:12 -0700663 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
664 rtc::Event* destruction_event,
665 int width,
666 int height) const {
667 VideoFrame frame =
668 VideoFrame::Builder()
669 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
670 destruction_event, width, height))
671 .set_timestamp_rtp(99)
672 .set_timestamp_ms(99)
673 .set_rotation(kVideoRotation_0)
674 .build();
675 frame.set_ntp_time_ms(ntp_time_ms);
676 return frame;
677 }
678
679 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
680 rtc::Event* destruction_event) const {
681 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
682 codec_height_);
683 }
684
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100685 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
686 MockBitrateObserver bitrate_observer;
687 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
688
689 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
690 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100692 DataRate::BitsPerSec(kTargetBitrateBps),
693 DataRate::BitsPerSec(kTargetBitrateBps),
694 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100695
696 video_source_.IncomingCapturedFrame(
697 CreateFrame(1, codec_width_, codec_height_));
698 WaitForEncodedFrame(1);
699 }
700
asapersson02465b82017-04-10 01:12:52 -0700701 void VerifyNoLimitation(const rtc::VideoSinkWants& wants) {
asapersson02465b82017-04-10 01:12:52 -0700702 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700703 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
704 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700705 }
706
asapersson09f05612017-05-15 23:40:18 -0700707 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
708 const rtc::VideoSinkWants& wants2) {
709 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
710 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
711 }
712
Åsa Persson8c1bf952018-09-13 10:42:19 +0200713 void VerifyFpsMaxResolutionMax(const rtc::VideoSinkWants& wants) {
714 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
715 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
716 EXPECT_FALSE(wants.target_pixel_count);
717 }
718
asapersson09f05612017-05-15 23:40:18 -0700719 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
720 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200721 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700722 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
723 EXPECT_GT(wants1.max_pixel_count, 0);
724 }
725
726 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
727 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200728 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700729 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
730 }
731
asaperssonf7e294d2017-06-13 23:25:22 -0700732 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
733 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200734 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700735 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
736 }
737
738 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
739 const rtc::VideoSinkWants& wants2) {
740 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
741 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
742 }
743
744 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
745 const rtc::VideoSinkWants& wants2) {
746 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
747 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
748 }
749
750 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
751 const rtc::VideoSinkWants& wants2) {
752 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
753 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
754 EXPECT_GT(wants1.max_pixel_count, 0);
755 }
756
757 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
758 const rtc::VideoSinkWants& wants2) {
759 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
760 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
761 }
762
asapersson09f05612017-05-15 23:40:18 -0700763 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants,
764 int pixel_count) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200765 EXPECT_EQ(kDefaultFramerate, wants.max_framerate_fps);
asapersson02465b82017-04-10 01:12:52 -0700766 EXPECT_LT(wants.max_pixel_count, pixel_count);
767 EXPECT_GT(wants.max_pixel_count, 0);
asapersson09f05612017-05-15 23:40:18 -0700768 }
769
770 void VerifyFpsLtResolutionMax(const rtc::VideoSinkWants& wants, int fps) {
771 EXPECT_LT(wants.max_framerate_fps, fps);
772 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
773 EXPECT_FALSE(wants.target_pixel_count);
asapersson02465b82017-04-10 01:12:52 -0700774 }
775
asaperssonf7e294d2017-06-13 23:25:22 -0700776 void VerifyFpsEqResolutionMax(const rtc::VideoSinkWants& wants,
777 int expected_fps) {
778 EXPECT_EQ(expected_fps, wants.max_framerate_fps);
779 EXPECT_EQ(std::numeric_limits<int>::max(), wants.max_pixel_count);
780 EXPECT_FALSE(wants.target_pixel_count);
781 }
782
Jonathan Yubc771b72017-12-08 17:04:29 -0800783 void VerifyBalancedModeFpsRange(const rtc::VideoSinkWants& wants,
784 int last_frame_pixels) {
785 // Balanced mode should always scale FPS to the desired range before
786 // attempting to scale resolution.
787 int fps_limit = wants.max_framerate_fps;
788 if (last_frame_pixels <= 320 * 240) {
Henrik Boström60383832020-02-28 09:03:53 +0100789 EXPECT_LE(7, fps_limit);
790 EXPECT_LE(fps_limit, 10);
Jonathan Yubc771b72017-12-08 17:04:29 -0800791 } else if (last_frame_pixels <= 480 * 270) {
Henrik Boström60383832020-02-28 09:03:53 +0100792 EXPECT_LE(10, fps_limit);
793 EXPECT_LE(fps_limit, 15);
Jonathan Yubc771b72017-12-08 17:04:29 -0800794 } else if (last_frame_pixels <= 640 * 480) {
795 EXPECT_LE(15, fps_limit);
796 } else {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200797 EXPECT_EQ(kDefaultFramerate, fps_limit);
Jonathan Yubc771b72017-12-08 17:04:29 -0800798 }
799 }
800
sprang4847ae62017-06-27 07:06:52 -0700801 void WaitForEncodedFrame(int64_t expected_ntp_time) {
802 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100803 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700804 }
805
806 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
807 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100808 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700809 return ok;
810 }
811
812 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
813 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100814 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700815 }
816
817 void ExpectDroppedFrame() {
818 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100819 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700820 }
821
822 bool WaitForFrame(int64_t timeout_ms) {
823 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100824 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700825 return ok;
826 }
827
perkj26091b12016-09-01 01:17:40 -0700828 class TestEncoder : public test::FakeEncoder {
829 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100830 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700831
asaperssonfab67072017-04-04 05:51:49 -0700832 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800833 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700834 return config_;
835 }
836
837 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800838 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700839 block_next_encode_ = true;
840 }
841
Erik Språngaed30702018-11-05 12:57:17 +0100842 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800843 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100845 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100846 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100847 info.scaling_settings = VideoEncoder::ScalingSettings(
848 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100849 }
850 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100851 for (int i = 0; i < kMaxSpatialLayers; ++i) {
852 if (temporal_layers_supported_[i]) {
853 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
854 info.fps_allocation[i].resize(num_layers);
855 }
856 }
Erik Språngaed30702018-11-05 12:57:17 +0100857 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200858
859 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100860 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100861 return info;
kthelgason876222f2016-11-29 01:44:11 -0800862 }
863
Erik Språngb7cb7b52019-02-26 15:52:33 +0100864 int32_t RegisterEncodeCompleteCallback(
865 EncodedImageCallback* callback) override {
866 rtc::CritScope lock(&local_crit_sect_);
867 encoded_image_callback_ = callback;
868 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
869 }
870
perkjfa10b552016-10-02 23:45:26 -0700871 void ContinueEncode() { continue_encode_event_.Set(); }
872
873 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
874 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800875 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700876 EXPECT_EQ(timestamp_, timestamp);
877 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
878 }
879
kthelgason2fc52542017-03-03 00:24:41 -0800880 void SetQualityScaling(bool b) {
881 rtc::CritScope lock(&local_crit_sect_);
882 quality_scaling_ = b;
883 }
kthelgasonad9010c2017-02-14 00:46:51 -0800884
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100885 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
886 rtc::CritScope lock(&local_crit_sect_);
887 requested_resolution_alignment_ = requested_resolution_alignment;
888 }
889
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100890 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
891 rtc::CritScope lock(&local_crit_sect_);
892 is_hardware_accelerated_ = is_hardware_accelerated;
893 }
894
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100895 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
896 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
897 rtc::CritScope lock(&local_crit_sect_);
898 temporal_layers_supported_[spatial_idx] = supported;
899 }
900
Sergey Silkin6456e352019-07-08 17:56:40 +0200901 void SetResolutionBitrateLimits(
902 std::vector<ResolutionBitrateLimits> thresholds) {
903 rtc::CritScope cs(&local_crit_sect_);
904 resolution_bitrate_limits_ = thresholds;
905 }
906
sprangfe627f32017-03-29 08:24:59 -0700907 void ForceInitEncodeFailure(bool force_failure) {
908 rtc::CritScope lock(&local_crit_sect_);
909 force_init_encode_failed_ = force_failure;
910 }
911
Niels Möller6bb5ab92019-01-11 11:11:10 +0100912 void SimulateOvershoot(double rate_factor) {
913 rtc::CritScope lock(&local_crit_sect_);
914 rate_factor_ = rate_factor;
915 }
916
Erik Språngd7329ca2019-02-21 21:19:53 +0100917 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100918 rtc::CritScope lock(&local_crit_sect_);
919 return last_framerate_;
920 }
921
Erik Språngd7329ca2019-02-21 21:19:53 +0100922 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100923 rtc::CritScope lock(&local_crit_sect_);
924 return last_update_rect_;
925 }
926
Niels Möller87e2d782019-03-07 10:18:23 +0100927 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100928 rtc::CritScope lock(&local_crit_sect_);
929 return last_frame_types_;
930 }
931
932 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100933 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100934 keyframe ? VideoFrameType::kVideoFrameKey
935 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 {
937 rtc::CritScope lock(&local_crit_sect_);
938 last_frame_types_ = frame_type;
939 }
Niels Möllerb859b322019-03-07 12:40:01 +0100940 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100941 }
942
Erik Språngb7cb7b52019-02-26 15:52:33 +0100943 void InjectEncodedImage(const EncodedImage& image) {
944 rtc::CritScope lock(&local_crit_sect_);
945 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
946 }
947
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200948 void InjectEncodedImage(const EncodedImage& image,
949 const CodecSpecificInfo* codec_specific_info,
950 const RTPFragmentationHeader* fragmentation) {
951 rtc::CritScope lock(&local_crit_sect_);
952 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
953 fragmentation);
954 }
955
Erik Språngd7329ca2019-02-21 21:19:53 +0100956 void ExpectNullFrame() {
957 rtc::CritScope lock(&local_crit_sect_);
958 expect_null_frame_ = true;
959 }
960
Erik Språng5056af02019-09-02 15:53:11 +0200961 absl::optional<VideoEncoder::RateControlParameters>
962 GetAndResetLastRateControlSettings() {
963 auto settings = last_rate_control_settings_;
964 last_rate_control_settings_.reset();
965 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100966 }
967
Sergey Silkin5ee69672019-07-02 14:18:34 +0200968 int GetNumEncoderInitializations() const {
969 rtc::CritScope lock(&local_crit_sect_);
970 return num_encoder_initializations_;
971 }
972
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200973 int GetNumSetRates() const {
974 rtc::CritScope lock(&local_crit_sect_);
975 return num_set_rates_;
976 }
977
perkjfa10b552016-10-02 23:45:26 -0700978 private:
perkj26091b12016-09-01 01:17:40 -0700979 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100980 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700981 bool block_encode;
982 {
brandtre78d2662017-01-16 05:57:16 -0800983 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100984 if (expect_null_frame_) {
985 EXPECT_EQ(input_image.timestamp(), 0u);
986 EXPECT_EQ(input_image.width(), 1);
987 last_frame_types_ = *frame_types;
988 expect_null_frame_ = false;
989 } else {
990 EXPECT_GT(input_image.timestamp(), timestamp_);
991 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
992 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
993 }
perkj26091b12016-09-01 01:17:40 -0700994
995 timestamp_ = input_image.timestamp();
996 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700997 last_input_width_ = input_image.width();
998 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700999 block_encode = block_next_encode_;
1000 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001001 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001002 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -07001003 }
Niels Möllerb859b322019-03-07 12:40:01 +01001004 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001005 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001006 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001007 return result;
1008 }
1009
sprangfe627f32017-03-29 08:24:59 -07001010 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001011 const Settings& settings) override {
1012 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001013
sprangfe627f32017-03-29 08:24:59 -07001014 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001015 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001016
1017 ++num_encoder_initializations_;
1018
Erik Språng82fad3d2018-03-21 09:57:23 +01001019 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001020 // Simulate setting up temporal layers, in order to validate the life
1021 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001022 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001023 frame_buffer_controller_ =
1024 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001025 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001026 if (force_init_encode_failed_) {
1027 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001028 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001029 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001030
Erik Språngb7cb7b52019-02-26 15:52:33 +01001031 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001032 return res;
1033 }
1034
Erik Språngb7cb7b52019-02-26 15:52:33 +01001035 int32_t Release() override {
1036 rtc::CritScope lock(&local_crit_sect_);
1037 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1038 initialized_ = EncoderState::kUninitialized;
1039 return FakeEncoder::Release();
1040 }
1041
Erik Språng16cb8f52019-04-12 13:59:09 +02001042 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001043 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001044 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001045 VideoBitrateAllocation adjusted_rate_allocation;
1046 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1047 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001048 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001049 adjusted_rate_allocation.SetBitrate(
1050 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001051 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001052 rate_factor_));
1053 }
1054 }
1055 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001056 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001057 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001058 RateControlParameters adjusted_paramters = parameters;
1059 adjusted_paramters.bitrate = adjusted_rate_allocation;
1060 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001061 }
1062
brandtre78d2662017-01-16 05:57:16 -08001063 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001064 enum class EncoderState {
1065 kUninitialized,
1066 kInitializationFailed,
1067 kInitialized
1068 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
1069 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -07001070 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -07001071 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -07001072 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1073 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1074 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1075 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1076 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001077 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001078 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +01001079 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -07001080 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001081 absl::optional<bool>
1082 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
1083 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -07001084 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001085 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
1086 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001087 absl::optional<VideoEncoder::RateControlParameters>
1088 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001089 VideoFrame::UpdateRect last_update_rect_
1090 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001091 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001092 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001093 EncodedImageCallback* encoded_image_callback_
1094 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001095 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001096 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001097 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1098 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001099 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001100 };
1101
mflodmancc3d4422017-08-03 08:27:51 -07001102 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001103 public:
1104 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001105 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001106
perkj26091b12016-09-01 01:17:40 -07001107 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001108 EXPECT_TRUE(
1109 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1110 }
1111
1112 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1113 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001114 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001115 if (!encoded_frame_event_.Wait(timeout_ms))
1116 return false;
perkj26091b12016-09-01 01:17:40 -07001117 {
1118 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001119 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001120 }
1121 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001122 return true;
perkj26091b12016-09-01 01:17:40 -07001123 }
1124
sprangb1ca0732017-02-01 08:38:12 -08001125 void WaitForEncodedFrame(uint32_t expected_width,
1126 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001127 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001128 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001129 }
1130
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001131 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001132 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001133 uint32_t width = 0;
1134 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001135 {
1136 rtc::CritScope lock(&crit_);
1137 width = last_width_;
1138 height = last_height_;
1139 }
1140 EXPECT_EQ(expected_height, height);
1141 EXPECT_EQ(expected_width, width);
1142 }
1143
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001144 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1145 int width = 0;
1146 int height = 0;
1147 {
1148 rtc::CritScope lock(&crit_);
1149 width = last_width_;
1150 height = last_height_;
1151 }
1152 EXPECT_EQ(width % resolution_alignment, 0);
1153 EXPECT_EQ(height % resolution_alignment, 0);
1154 }
1155
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001156 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1157 VideoRotation rotation;
1158 {
1159 rtc::CritScope lock(&crit_);
1160 rotation = last_rotation_;
1161 }
1162 EXPECT_EQ(expected_rotation, rotation);
1163 }
1164
kthelgason2fc52542017-03-03 00:24:41 -08001165 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001166
sprangc5d62e22017-04-02 23:53:04 -07001167 bool WaitForFrame(int64_t timeout_ms) {
1168 return encoded_frame_event_.Wait(timeout_ms);
1169 }
1170
perkj26091b12016-09-01 01:17:40 -07001171 void SetExpectNoFrames() {
1172 rtc::CritScope lock(&crit_);
1173 expect_frames_ = false;
1174 }
1175
asaperssonfab67072017-04-04 05:51:49 -07001176 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001177 rtc::CritScope lock(&crit_);
1178 return number_of_reconfigurations_;
1179 }
1180
asaperssonfab67072017-04-04 05:51:49 -07001181 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001182 rtc::CritScope lock(&crit_);
1183 return min_transmit_bitrate_bps_;
1184 }
1185
Erik Språngd7329ca2019-02-21 21:19:53 +01001186 void SetNumExpectedLayers(size_t num_layers) {
1187 rtc::CritScope lock(&crit_);
1188 num_expected_layers_ = num_layers;
1189 }
1190
Erik Språngb7cb7b52019-02-26 15:52:33 +01001191 int64_t GetLastCaptureTimeMs() const {
1192 rtc::CritScope lock(&crit_);
1193 return last_capture_time_ms_;
1194 }
1195
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001196 std::vector<uint8_t> GetLastEncodedImageData() {
1197 rtc::CritScope lock(&crit_);
1198 return std::move(last_encoded_image_data_);
1199 }
1200
1201 RTPFragmentationHeader GetLastFragmentation() {
1202 rtc::CritScope lock(&crit_);
1203 return std::move(last_fragmentation_);
1204 }
1205
perkj26091b12016-09-01 01:17:40 -07001206 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001207 Result OnEncodedImage(
1208 const EncodedImage& encoded_image,
1209 const CodecSpecificInfo* codec_specific_info,
1210 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001211 rtc::CritScope lock(&crit_);
1212 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001213 last_encoded_image_data_ = std::vector<uint8_t>(
1214 encoded_image.data(), encoded_image.data() + encoded_image.size());
1215 if (fragmentation) {
1216 last_fragmentation_.CopyFrom(*fragmentation);
1217 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001218 uint32_t timestamp = encoded_image.Timestamp();
1219 if (last_timestamp_ != timestamp) {
1220 num_received_layers_ = 1;
1221 } else {
1222 ++num_received_layers_;
1223 }
1224 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001225 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001226 last_width_ = encoded_image._encodedWidth;
1227 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001228 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001229 if (num_received_layers_ == num_expected_layers_) {
1230 encoded_frame_event_.Set();
1231 }
sprangb1ca0732017-02-01 08:38:12 -08001232 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001233 }
1234
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001235 void OnEncoderConfigurationChanged(
1236 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001237 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001238 VideoEncoderConfig::ContentType content_type,
1239 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001240 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001241 ++number_of_reconfigurations_;
1242 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1243 }
1244
perkj26091b12016-09-01 01:17:40 -07001245 rtc::CriticalSection crit_;
1246 TestEncoder* test_encoder_;
1247 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001248 std::vector<uint8_t> last_encoded_image_data_;
1249 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001250 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001251 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001252 uint32_t last_height_ = 0;
1253 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001254 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001255 size_t num_expected_layers_ = 1;
1256 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001257 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001258 int number_of_reconfigurations_ = 0;
1259 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001260 };
1261
Sergey Silkin5ee69672019-07-02 14:18:34 +02001262 class VideoBitrateAllocatorProxyFactory
1263 : public VideoBitrateAllocatorFactory {
1264 public:
1265 VideoBitrateAllocatorProxyFactory()
1266 : bitrate_allocator_factory_(
1267 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1268
1269 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1270 const VideoCodec& codec) override {
1271 rtc::CritScope lock(&crit_);
1272 codec_config_ = codec;
1273 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1274 }
1275
1276 VideoCodec codec_config() const {
1277 rtc::CritScope lock(&crit_);
1278 return codec_config_;
1279 }
1280
1281 private:
1282 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1283
1284 rtc::CriticalSection crit_;
1285 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1286 };
1287
perkj26091b12016-09-01 01:17:40 -07001288 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001289 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001290 int codec_width_;
1291 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001292 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001293 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001294 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001295 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001296 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001297 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001298 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001299 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001300 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001301 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001302};
1303
mflodmancc3d4422017-08-03 08:27:51 -07001304TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001306 DataRate::BitsPerSec(kTargetBitrateBps),
1307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001309 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001310 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001311 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001312 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001313 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001314}
1315
mflodmancc3d4422017-08-03 08:27:51 -07001316TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001317 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001318 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001319 // The encoder will cache up to one frame for a short duration. Adding two
1320 // frames means that the first frame will be dropped and the second frame will
1321 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001322 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001323 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001324 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001325
Henrik Boström381d1092020-05-12 18:49:07 +02001326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001327 DataRate::BitsPerSec(kTargetBitrateBps),
1328 DataRate::BitsPerSec(kTargetBitrateBps),
1329 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001330
Sebastian Janssona3177052018-04-10 13:05:49 +02001331 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001332 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001333 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1334
1335 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001336 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001337}
1338
mflodmancc3d4422017-08-03 08:27:51 -07001339TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001340 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001341 DataRate::BitsPerSec(kTargetBitrateBps),
1342 DataRate::BitsPerSec(kTargetBitrateBps),
1343 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001344 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001345 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001346
Henrik Boström381d1092020-05-12 18:49:07 +02001347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1348 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1349 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001350 // The encoder will cache up to one frame for a short duration. Adding two
1351 // frames means that the first frame will be dropped and the second frame will
1352 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001353 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001354 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001355
Henrik Boström381d1092020-05-12 18:49:07 +02001356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001357 DataRate::BitsPerSec(kTargetBitrateBps),
1358 DataRate::BitsPerSec(kTargetBitrateBps),
1359 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001360 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001361 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1362 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001363 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001364}
1365
mflodmancc3d4422017-08-03 08:27:51 -07001366TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001367 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001368 DataRate::BitsPerSec(kTargetBitrateBps),
1369 DataRate::BitsPerSec(kTargetBitrateBps),
1370 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001371 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001372 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001373
1374 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001375 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001376
perkja49cbd32016-09-16 07:53:41 -07001377 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001378 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001379 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001380}
1381
mflodmancc3d4422017-08-03 08:27:51 -07001382TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001384 DataRate::BitsPerSec(kTargetBitrateBps),
1385 DataRate::BitsPerSec(kTargetBitrateBps),
1386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001387
perkja49cbd32016-09-16 07:53:41 -07001388 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001389 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001390
mflodmancc3d4422017-08-03 08:27:51 -07001391 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001392 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001393 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001394 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1395 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001396}
1397
mflodmancc3d4422017-08-03 08:27:51 -07001398TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001400 DataRate::BitsPerSec(kTargetBitrateBps),
1401 DataRate::BitsPerSec(kTargetBitrateBps),
1402 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001403
1404 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001405 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001406 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001407 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1408 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001409 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1410 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001411 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001412 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001413
mflodmancc3d4422017-08-03 08:27:51 -07001414 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001415}
1416
Noah Richards51db4212019-06-12 06:59:12 -07001417TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001418 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001419 DataRate::BitsPerSec(kTargetBitrateBps),
1420 DataRate::BitsPerSec(kTargetBitrateBps),
1421 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001422
1423 rtc::Event frame_destroyed_event;
1424 video_source_.IncomingCapturedFrame(
1425 CreateFakeNativeFrame(1, &frame_destroyed_event));
1426 ExpectDroppedFrame();
1427 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1428 video_stream_encoder_->Stop();
1429}
1430
1431TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1432 // Use the cropping factory.
1433 video_encoder_config_.video_stream_factory =
1434 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1435 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1436 kMaxPayloadLength);
1437 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1438
1439 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001440 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001441 DataRate::BitsPerSec(kTargetBitrateBps),
1442 DataRate::BitsPerSec(kTargetBitrateBps),
1443 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001444 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1445 WaitForEncodedFrame(1);
1446 // The encoder will have been configured once.
1447 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1448 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1449 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1450
1451 // Now send in a fake frame that needs to be cropped as the width/height
1452 // aren't divisible by 4 (see CreateEncoderStreams above).
1453 rtc::Event frame_destroyed_event;
1454 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1455 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1456 ExpectDroppedFrame();
1457 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1458 video_stream_encoder_->Stop();
1459}
1460
Ying Wang9b881ab2020-02-07 14:29:32 +01001461TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001463 DataRate::BitsPerSec(kTargetBitrateBps),
1464 DataRate::BitsPerSec(kTargetBitrateBps),
1465 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001466 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1467 WaitForEncodedFrame(1);
1468
Henrik Boström381d1092020-05-12 18:49:07 +02001469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001470 DataRate::BitsPerSec(kTargetBitrateBps),
1471 DataRate::BitsPerSec(kTargetBitrateBps),
1472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001473 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1474 // frames. Adding two frames means that the first frame will be dropped and
1475 // the second frame will be sent to the encoder.
1476 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1477 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1478 WaitForEncodedFrame(3);
1479 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1480 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1481 WaitForEncodedFrame(5);
1482 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1483 video_stream_encoder_->Stop();
1484}
1485
mflodmancc3d4422017-08-03 08:27:51 -07001486TEST_F(VideoStreamEncoderTest,
1487 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001489 DataRate::BitsPerSec(kTargetBitrateBps),
1490 DataRate::BitsPerSec(kTargetBitrateBps),
1491 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001492 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001493
1494 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001495 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001496 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001497 // The encoder will have been configured once when the first frame is
1498 // received.
1499 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001500
1501 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001502 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001503 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001504 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001505 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001506
1507 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001508 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001509 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001510 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001511 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001512
mflodmancc3d4422017-08-03 08:27:51 -07001513 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001514}
1515
mflodmancc3d4422017-08-03 08:27:51 -07001516TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001518 DataRate::BitsPerSec(kTargetBitrateBps),
1519 DataRate::BitsPerSec(kTargetBitrateBps),
1520 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001521
1522 // Capture a frame and wait for it to synchronize with the encoder thread.
1523 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001524 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001525 // The encoder will have been configured once.
1526 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001527 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1528 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1529
1530 codec_width_ *= 2;
1531 codec_height_ *= 2;
1532 // Capture a frame with a higher resolution and wait for it to synchronize
1533 // with the encoder thread.
1534 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001535 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001536 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1537 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001538 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001539
mflodmancc3d4422017-08-03 08:27:51 -07001540 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001541}
1542
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001543TEST_F(VideoStreamEncoderTest,
1544 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001546 DataRate::BitsPerSec(kTargetBitrateBps),
1547 DataRate::BitsPerSec(kTargetBitrateBps),
1548 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001549
1550 // Capture a frame and wait for it to synchronize with the encoder thread.
1551 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1552 WaitForEncodedFrame(1);
1553
1554 VideoEncoderConfig video_encoder_config;
1555 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1556 // Changing the max payload data length recreates encoder.
1557 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1558 kMaxPayloadLength / 2);
1559
1560 // Capture a frame and wait for it to synchronize with the encoder thread.
1561 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1562 WaitForEncodedFrame(2);
1563 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1564
1565 video_stream_encoder_->Stop();
1566}
1567
Sergey Silkin5ee69672019-07-02 14:18:34 +02001568TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001570 DataRate::BitsPerSec(kTargetBitrateBps),
1571 DataRate::BitsPerSec(kTargetBitrateBps),
1572 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001573
1574 VideoEncoderConfig video_encoder_config;
1575 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1576 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1577 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1578 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1579 kMaxPayloadLength);
1580
1581 // Capture a frame and wait for it to synchronize with the encoder thread.
1582 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1583 WaitForEncodedFrame(1);
1584 // The encoder will have been configured once when the first frame is
1585 // received.
1586 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1587 EXPECT_EQ(kTargetBitrateBps,
1588 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1589 EXPECT_EQ(kStartBitrateBps,
1590 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1591
Sergey Silkin6456e352019-07-08 17:56:40 +02001592 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1593 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001594 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1595 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1596 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1597 kMaxPayloadLength);
1598
1599 // Capture a frame and wait for it to synchronize with the encoder thread.
1600 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1601 WaitForEncodedFrame(2);
1602 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1603 // Bitrate limits have changed - rate allocator should be reconfigured,
1604 // encoder should not be reconfigured.
1605 EXPECT_EQ(kTargetBitrateBps * 2,
1606 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1607 EXPECT_EQ(kStartBitrateBps * 2,
1608 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1609 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1610
1611 video_stream_encoder_->Stop();
1612}
1613
Sergey Silkin6456e352019-07-08 17:56:40 +02001614TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001615 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001617 DataRate::BitsPerSec(kTargetBitrateBps),
1618 DataRate::BitsPerSec(kTargetBitrateBps),
1619 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001620
Sergey Silkincd02eba2020-01-20 14:48:40 +01001621 const uint32_t kMinEncBitrateKbps = 100;
1622 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001623 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001624 /*frame_size_pixels=*/codec_width_ * codec_height_,
1625 /*min_start_bitrate_bps=*/0,
1626 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1627 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001628 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1629
Sergey Silkincd02eba2020-01-20 14:48:40 +01001630 VideoEncoderConfig video_encoder_config;
1631 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1632 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1633 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1634 (kMinEncBitrateKbps + 1) * 1000;
1635 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1636 kMaxPayloadLength);
1637
1638 // When both encoder and app provide bitrate limits, the intersection of
1639 // provided sets should be used.
1640 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1641 WaitForEncodedFrame(1);
1642 EXPECT_EQ(kMaxEncBitrateKbps,
1643 bitrate_allocator_factory_.codec_config().maxBitrate);
1644 EXPECT_EQ(kMinEncBitrateKbps + 1,
1645 bitrate_allocator_factory_.codec_config().minBitrate);
1646
1647 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1648 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1649 (kMinEncBitrateKbps - 1) * 1000;
1650 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1651 kMaxPayloadLength);
1652 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001653 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001654 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001655 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001656 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001657 bitrate_allocator_factory_.codec_config().minBitrate);
1658
Sergey Silkincd02eba2020-01-20 14:48:40 +01001659 video_stream_encoder_->Stop();
1660}
1661
1662TEST_F(VideoStreamEncoderTest,
1663 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001665 DataRate::BitsPerSec(kTargetBitrateBps),
1666 DataRate::BitsPerSec(kTargetBitrateBps),
1667 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001668
1669 const uint32_t kMinAppBitrateKbps = 100;
1670 const uint32_t kMaxAppBitrateKbps = 200;
1671 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1672 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1673 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1674 /*frame_size_pixels=*/codec_width_ * codec_height_,
1675 /*min_start_bitrate_bps=*/0,
1676 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1677 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1678 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1679
1680 VideoEncoderConfig video_encoder_config;
1681 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1682 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1683 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1684 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001685 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1686 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001687
Sergey Silkincd02eba2020-01-20 14:48:40 +01001688 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1689 WaitForEncodedFrame(1);
1690 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001691 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001692 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001693 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001694
1695 video_stream_encoder_->Stop();
1696}
1697
1698TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001699 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001700 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001701 DataRate::BitsPerSec(kTargetBitrateBps),
1702 DataRate::BitsPerSec(kTargetBitrateBps),
1703 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001704
1705 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001706 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001707 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001708 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001709 fake_encoder_.SetResolutionBitrateLimits(
1710 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1711
1712 VideoEncoderConfig video_encoder_config;
1713 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1714 video_encoder_config.max_bitrate_bps = 0;
1715 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1716 kMaxPayloadLength);
1717
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001718 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001719 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1720 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001721 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1722 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001723 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1724 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1725
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001726 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001727 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1728 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001729 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1730 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001731 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1732 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1733
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001734 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001735 // encoder for 360p should be used.
1736 video_source_.IncomingCapturedFrame(
1737 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1738 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001739 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1740 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001741 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1742 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1743
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001744 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001745 // ignored.
1746 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1747 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001748 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1749 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001750 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1751 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001752 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1753 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001754 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1755 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1756
1757 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1758 // for 270p should be used.
1759 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1760 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001761 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1762 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001763 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1764 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1765
1766 video_stream_encoder_->Stop();
1767}
1768
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001769TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001770 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001771 DataRate::BitsPerSec(kTargetBitrateBps),
1772 DataRate::BitsPerSec(kTargetBitrateBps),
1773 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001774
1775 VideoEncoderConfig video_encoder_config;
1776 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1777 video_encoder_config.max_bitrate_bps = 0;
1778 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1779 kMaxPayloadLength);
1780
1781 // Encode 720p frame to get the default encoder target bitrate.
1782 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1783 WaitForEncodedFrame(1);
1784 const uint32_t kDefaultTargetBitrateFor720pKbps =
1785 bitrate_allocator_factory_.codec_config()
1786 .simulcastStream[0]
1787 .targetBitrate;
1788
1789 // Set the max recommended encoder bitrate to something lower than the default
1790 // target bitrate.
1791 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1792 1280 * 720, 10 * 1000, 10 * 1000,
1793 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1794 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1795
1796 // Change resolution to trigger encoder reinitialization.
1797 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1798 WaitForEncodedFrame(2);
1799 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1800 WaitForEncodedFrame(3);
1801
1802 // Ensure the target bitrate is capped by the max bitrate.
1803 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1804 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1805 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1806 .simulcastStream[0]
1807 .targetBitrate *
1808 1000,
1809 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1810
1811 video_stream_encoder_->Stop();
1812}
1813
mflodmancc3d4422017-08-03 08:27:51 -07001814TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001815 EXPECT_TRUE(video_source_.has_sinks());
1816 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001817 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001818 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001819 EXPECT_FALSE(video_source_.has_sinks());
1820 EXPECT_TRUE(new_video_source.has_sinks());
1821
mflodmancc3d4422017-08-03 08:27:51 -07001822 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001823}
1824
mflodmancc3d4422017-08-03 08:27:51 -07001825TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001826 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001827 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001828 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001829 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001830}
1831
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001832TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1833 constexpr int kRequestedResolutionAlignment = 7;
1834 video_source_.set_adaptation_enabled(true);
1835 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001837 DataRate::BitsPerSec(kTargetBitrateBps),
1838 DataRate::BitsPerSec(kTargetBitrateBps),
1839 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001840
1841 // On the 1st frame, we should have initialized the encoder and
1842 // asked for its resolution requirements.
1843 video_source_.IncomingCapturedFrame(
1844 CreateFrame(1, codec_width_, codec_height_));
1845 WaitForEncodedFrame(1);
1846 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1847 kRequestedResolutionAlignment);
1848
1849 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1850 // (It's up the to the encoder to potentially drop the previous frame,
1851 // to avoid coding back-to-back keyframes.)
1852 video_source_.IncomingCapturedFrame(
1853 CreateFrame(2, codec_width_, codec_height_));
1854 WaitForEncodedFrame(2);
1855 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1856
1857 video_stream_encoder_->Stop();
1858}
1859
Jonathan Yubc771b72017-12-08 17:04:29 -08001860TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1861 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001862 const int kWidth = 1280;
1863 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001864
1865 // We rely on the automatic resolution adaptation, but we handle framerate
1866 // adaptation manually by mocking the stats proxy.
1867 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001868
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001869 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001871 DataRate::BitsPerSec(kTargetBitrateBps),
1872 DataRate::BitsPerSec(kTargetBitrateBps),
1873 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001874 video_stream_encoder_->SetSource(&video_source_,
1875 webrtc::DegradationPreference::BALANCED);
Jonathan Yubc771b72017-12-08 17:04:29 -08001876 VerifyNoLimitation(video_source_.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07001877 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001878 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001879 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1880
Jonathan Yubc771b72017-12-08 17:04:29 -08001881 // Adapt down as far as possible.
1882 rtc::VideoSinkWants last_wants;
1883 int64_t t = 1;
1884 int loop_count = 0;
1885 do {
1886 ++loop_count;
1887 last_wants = video_source_.sink_wants();
1888
1889 // Simulate the framerate we've been asked to adapt to.
1890 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1891 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1892 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1893 mock_stats.input_frame_rate = fps;
1894 stats_proxy_->SetMockStats(mock_stats);
1895
1896 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1897 sink_.WaitForEncodedFrame(t);
1898 t += frame_interval_ms;
1899
mflodmancc3d4422017-08-03 08:27:51 -07001900 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001901 VerifyBalancedModeFpsRange(
1902 video_source_.sink_wants(),
1903 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1904 } while (video_source_.sink_wants().max_pixel_count <
1905 last_wants.max_pixel_count ||
1906 video_source_.sink_wants().max_framerate_fps <
1907 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001908
Jonathan Yubc771b72017-12-08 17:04:29 -08001909 // Verify that we've adapted all the way down.
1910 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001911 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001912 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1913 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001914 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001915 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1916 *video_source_.last_sent_height());
1917 EXPECT_EQ(kMinBalancedFramerateFps,
1918 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001919
Jonathan Yubc771b72017-12-08 17:04:29 -08001920 // Adapt back up the same number of times we adapted down.
1921 for (int i = 0; i < loop_count - 1; ++i) {
1922 last_wants = video_source_.sink_wants();
1923
1924 // Simulate the framerate we've been asked to adapt to.
1925 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1926 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1927 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1928 mock_stats.input_frame_rate = fps;
1929 stats_proxy_->SetMockStats(mock_stats);
1930
1931 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1932 sink_.WaitForEncodedFrame(t);
1933 t += frame_interval_ms;
1934
Henrik Boström91aa7322020-04-28 12:24:33 +02001935 video_stream_encoder_->TriggerCpuUnderuse();
Jonathan Yubc771b72017-12-08 17:04:29 -08001936 VerifyBalancedModeFpsRange(
1937 video_source_.sink_wants(),
1938 *video_source_.last_sent_width() * *video_source_.last_sent_height());
1939 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1940 last_wants.max_pixel_count ||
1941 video_source_.sink_wants().max_framerate_fps >
1942 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001943 }
1944
Åsa Persson8c1bf952018-09-13 10:42:19 +02001945 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08001946 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001947 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001948 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1949 EXPECT_EQ((loop_count - 1) * 2,
1950 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001951
mflodmancc3d4422017-08-03 08:27:51 -07001952 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001953}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001954
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001955TEST_F(VideoStreamEncoderTest,
1956 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1957 video_stream_encoder_->OnBitrateUpdated(
1958 DataRate::BitsPerSec(kTargetBitrateBps),
1959 DataRate::BitsPerSec(kTargetBitrateBps),
1960 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1961 VerifyNoLimitation(video_source_.sink_wants());
1962
1963 const int kFrameWidth = 1280;
1964 const int kFrameHeight = 720;
1965
1966 int64_t ntp_time = kFrameIntervalMs;
1967
1968 // Force an input frame rate to be available, or the adaptation call won't
1969 // know what framerate to adapt form.
1970 const int kInputFps = 30;
1971 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1972 stats.input_frame_rate = kInputFps;
1973 stats_proxy_->SetMockStats(stats);
1974
1975 video_source_.set_adaptation_enabled(true);
1976 video_stream_encoder_->SetSource(
1977 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
1978 VerifyNoLimitation(video_source_.sink_wants());
1979 video_source_.IncomingCapturedFrame(
1980 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1981 sink_.WaitForEncodedFrame(ntp_time);
1982 ntp_time += kFrameIntervalMs;
1983
1984 // Trigger CPU overuse.
1985 video_stream_encoder_->TriggerCpuOveruse();
1986 video_source_.IncomingCapturedFrame(
1987 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1988 sink_.WaitForEncodedFrame(ntp_time);
1989 ntp_time += kFrameIntervalMs;
1990
1991 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1992 EXPECT_EQ(std::numeric_limits<int>::max(),
1993 video_source_.sink_wants().max_pixel_count);
1994 // Some framerate constraint should be set.
1995 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1996 EXPECT_LT(restricted_fps, kInputFps);
1997 video_source_.IncomingCapturedFrame(
1998 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1999 sink_.WaitForEncodedFrame(ntp_time);
2000 ntp_time += 100;
2001
Henrik Boström2671dac2020-05-19 16:29:09 +02002002 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002003 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2004 // Give the encoder queue time to process the change in degradation preference
2005 // by waiting for an encoded frame.
2006 video_source_.IncomingCapturedFrame(
2007 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2008 sink_.WaitForEncodedFrame(ntp_time);
2009 ntp_time += kFrameIntervalMs;
2010
2011 video_stream_encoder_->TriggerQualityLow();
2012 video_source_.IncomingCapturedFrame(
2013 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2014 sink_.WaitForEncodedFrame(ntp_time);
2015 ntp_time += kFrameIntervalMs;
2016
2017 // Some resolution constraint should be set.
2018 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2019 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2020 kFrameWidth * kFrameHeight);
2021 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2022
2023 int pixel_count = video_source_.sink_wants().max_pixel_count;
2024 // Triggering a CPU underuse should not change the sink wants since it has
2025 // not been overused for resolution since we changed degradation preference.
2026 video_stream_encoder_->TriggerCpuUnderuse();
2027 video_source_.IncomingCapturedFrame(
2028 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2029 sink_.WaitForEncodedFrame(ntp_time);
2030 ntp_time += kFrameIntervalMs;
2031 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2032 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2033
2034 // Change the degradation preference back. CPU underuse should now adapt.
Henrik Boström2671dac2020-05-19 16:29:09 +02002035 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002036 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2037 video_source_.IncomingCapturedFrame(
2038 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2039 sink_.WaitForEncodedFrame(ntp_time);
2040 ntp_time += 100;
2041 // Resolution adaptations is gone after changing degradation preference.
2042 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2043 EXPECT_EQ(std::numeric_limits<int>::max(),
2044 video_source_.sink_wants().max_pixel_count);
2045 // The fps adaptation from above is now back.
2046 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2047
2048 // Trigger CPU underuse.
2049 video_stream_encoder_->TriggerCpuUnderuse();
2050 video_source_.IncomingCapturedFrame(
2051 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2052 sink_.WaitForEncodedFrame(ntp_time);
2053 ntp_time += kFrameIntervalMs;
2054 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2055
2056 video_stream_encoder_->Stop();
2057}
2058
mflodmancc3d4422017-08-03 08:27:51 -07002059TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002061 DataRate::BitsPerSec(kTargetBitrateBps),
2062 DataRate::BitsPerSec(kTargetBitrateBps),
2063 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002064 VerifyNoLimitation(video_source_.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002065
sprangc5d62e22017-04-02 23:53:04 -07002066 const int kFrameWidth = 1280;
2067 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002068
Åsa Persson8c1bf952018-09-13 10:42:19 +02002069 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002070
kthelgason5e13d412016-12-01 03:59:51 -08002071 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002072 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002073 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002074 frame_timestamp += kFrameIntervalMs;
2075
perkj803d97f2016-11-01 11:45:46 -07002076 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002077 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002078 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002079 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002080 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002081 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002082
asapersson0944a802017-04-07 00:57:58 -07002083 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002084 // wanted resolution.
2085 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2086 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2087 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002088 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002089
2090 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002091 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002092 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002093 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002094 // Give the encoder queue time to process the change in degradation preference
2095 // by waiting for an encoded frame.
2096 new_video_source.IncomingCapturedFrame(
2097 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2098 sink_.WaitForEncodedFrame(frame_timestamp);
2099 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002100 // Initially no degradation registered.
Åsa Persson8c1bf952018-09-13 10:42:19 +02002101 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002102
sprangc5d62e22017-04-02 23:53:04 -07002103 // Force an input frame rate to be available, or the adaptation call won't
2104 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002105 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002106 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002107 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002108 stats_proxy_->SetMockStats(stats);
2109
mflodmancc3d4422017-08-03 08:27:51 -07002110 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002111 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002112 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002113 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002114 frame_timestamp += kFrameIntervalMs;
2115
2116 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002117 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002118 EXPECT_EQ(std::numeric_limits<int>::max(),
2119 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002120 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002121
asapersson02465b82017-04-10 01:12:52 -07002122 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002123 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2124 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002125 // Give the encoder queue time to process the change in degradation preference
2126 // by waiting for an encoded frame.
2127 new_video_source.IncomingCapturedFrame(
2128 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2129 sink_.WaitForEncodedFrame(frame_timestamp);
2130 frame_timestamp += kFrameIntervalMs;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002131 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
sprangc5d62e22017-04-02 23:53:04 -07002132
mflodmancc3d4422017-08-03 08:27:51 -07002133 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002134 new_video_source.IncomingCapturedFrame(
2135 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002136 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002137 frame_timestamp += kFrameIntervalMs;
2138
2139 // Still no degradation.
Åsa Persson8c1bf952018-09-13 10:42:19 +02002140 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
perkj803d97f2016-11-01 11:45:46 -07002141
2142 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002143 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002144 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002145 // Give the encoder queue time to process the change in degradation preference
2146 // by waiting for an encoded frame.
2147 new_video_source.IncomingCapturedFrame(
2148 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2149 sink_.WaitForEncodedFrame(frame_timestamp);
2150 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002151 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2152 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002153 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002154 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002155
2156 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002157 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002158 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002159 // Give the encoder queue time to process the change in degradation preference
2160 // by waiting for an encoded frame.
2161 new_video_source.IncomingCapturedFrame(
2162 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2163 sink_.WaitForEncodedFrame(frame_timestamp);
2164 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002165 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2166 EXPECT_EQ(std::numeric_limits<int>::max(),
2167 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002168 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002169
mflodmancc3d4422017-08-03 08:27:51 -07002170 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002171}
2172
mflodmancc3d4422017-08-03 08:27:51 -07002173TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002175 DataRate::BitsPerSec(kTargetBitrateBps),
2176 DataRate::BitsPerSec(kTargetBitrateBps),
2177 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002178
asaperssonfab67072017-04-04 05:51:49 -07002179 const int kWidth = 1280;
2180 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002181 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002182 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002183 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2184 EXPECT_FALSE(stats.bw_limited_resolution);
2185 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2186
2187 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002188 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002189 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002190 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002191
2192 stats = stats_proxy_->GetStats();
2193 EXPECT_TRUE(stats.bw_limited_resolution);
2194 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2195
2196 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002197 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002198 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002199 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002200
2201 stats = stats_proxy_->GetStats();
2202 EXPECT_FALSE(stats.bw_limited_resolution);
2203 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2204 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2205
mflodmancc3d4422017-08-03 08:27:51 -07002206 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002207}
2208
mflodmancc3d4422017-08-03 08:27:51 -07002209TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002210 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002211 DataRate::BitsPerSec(kTargetBitrateBps),
2212 DataRate::BitsPerSec(kTargetBitrateBps),
2213 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002214
2215 const int kWidth = 1280;
2216 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002217 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002218 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002219 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2220 EXPECT_FALSE(stats.cpu_limited_resolution);
2221 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2222
2223 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002224 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002225 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002226 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002227
2228 stats = stats_proxy_->GetStats();
2229 EXPECT_TRUE(stats.cpu_limited_resolution);
2230 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2231
2232 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002233 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002234 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002235 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002236
2237 stats = stats_proxy_->GetStats();
2238 EXPECT_FALSE(stats.cpu_limited_resolution);
2239 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002240 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002241
mflodmancc3d4422017-08-03 08:27:51 -07002242 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002243}
2244
mflodmancc3d4422017-08-03 08:27:51 -07002245TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002246 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002247 DataRate::BitsPerSec(kTargetBitrateBps),
2248 DataRate::BitsPerSec(kTargetBitrateBps),
2249 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002250
asaperssonfab67072017-04-04 05:51:49 -07002251 const int kWidth = 1280;
2252 const int kHeight = 720;
2253 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002254 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002255 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002256 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002257 EXPECT_FALSE(stats.cpu_limited_resolution);
2258 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2259
asaperssonfab67072017-04-04 05:51:49 -07002260 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002261 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002262 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002263 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002264 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002265 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002266 EXPECT_TRUE(stats.cpu_limited_resolution);
2267 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2268
2269 // Set new source with adaptation still enabled.
2270 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002271 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002272 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002273
asaperssonfab67072017-04-04 05:51:49 -07002274 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002275 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002276 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002277 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002278 EXPECT_TRUE(stats.cpu_limited_resolution);
2279 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2280
2281 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002282 video_stream_encoder_->SetSource(&new_video_source,
2283 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002284
asaperssonfab67072017-04-04 05:51:49 -07002285 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002286 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002287 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002288 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002289 EXPECT_FALSE(stats.cpu_limited_resolution);
2290 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2291
2292 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002293 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002294 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002295
asaperssonfab67072017-04-04 05:51:49 -07002296 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002297 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002298 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002299 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002300 EXPECT_TRUE(stats.cpu_limited_resolution);
2301 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2302
asaperssonfab67072017-04-04 05:51:49 -07002303 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002304 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002305 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002306 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002307 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002308 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002309 EXPECT_FALSE(stats.cpu_limited_resolution);
2310 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002311 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002312
mflodmancc3d4422017-08-03 08:27:51 -07002313 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002314}
2315
mflodmancc3d4422017-08-03 08:27:51 -07002316TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002318 DataRate::BitsPerSec(kTargetBitrateBps),
2319 DataRate::BitsPerSec(kTargetBitrateBps),
2320 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002321
asaperssonfab67072017-04-04 05:51:49 -07002322 const int kWidth = 1280;
2323 const int kHeight = 720;
2324 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002325 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002326 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002327 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002328 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002329 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002330
2331 // Set new source with adaptation still enabled.
2332 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002333 video_stream_encoder_->SetSource(&new_video_source,
2334 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002335
asaperssonfab67072017-04-04 05:51:49 -07002336 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002337 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002338 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002339 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002340 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002341 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002342
asaperssonfab67072017-04-04 05:51:49 -07002343 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002344 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002345 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002346 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002347 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002348 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002349 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002350 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002351
asaperssonfab67072017-04-04 05:51:49 -07002352 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002353 video_stream_encoder_->SetSource(&new_video_source,
2354 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002355
asaperssonfab67072017-04-04 05:51:49 -07002356 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002357 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002358 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002359 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002360 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002361 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002362
asapersson02465b82017-04-10 01:12:52 -07002363 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002364 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002365 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002366
asaperssonfab67072017-04-04 05:51:49 -07002367 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002368 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002369 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002370 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002371 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002372 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2373 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002374
mflodmancc3d4422017-08-03 08:27:51 -07002375 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002376}
2377
mflodmancc3d4422017-08-03 08:27:51 -07002378TEST_F(VideoStreamEncoderTest,
2379 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002380 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002381 DataRate::BitsPerSec(kTargetBitrateBps),
2382 DataRate::BitsPerSec(kTargetBitrateBps),
2383 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002384
2385 const int kWidth = 1280;
2386 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002387 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002388 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002389 video_source_.IncomingCapturedFrame(
2390 CreateFrame(timestamp_ms, kWidth, kHeight));
2391 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002392 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2393 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2394 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2395
2396 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002397 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002398 timestamp_ms += kFrameIntervalMs;
2399 video_source_.IncomingCapturedFrame(
2400 CreateFrame(timestamp_ms, kWidth, kHeight));
2401 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002402 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2403 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2404 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2405
2406 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002407 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002408 timestamp_ms += kFrameIntervalMs;
2409 video_source_.IncomingCapturedFrame(
2410 CreateFrame(timestamp_ms, kWidth, kHeight));
2411 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002412 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2413 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2414 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2415
Niels Möller4db138e2018-04-19 09:04:13 +02002416 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002417 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002418
2419 VideoEncoderConfig video_encoder_config;
2420 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2421 // Make format different, to force recreation of encoder.
2422 video_encoder_config.video_format.parameters["foo"] = "foo";
2423 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002424 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002425 timestamp_ms += kFrameIntervalMs;
2426 video_source_.IncomingCapturedFrame(
2427 CreateFrame(timestamp_ms, kWidth, kHeight));
2428 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002429 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2430 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2431 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2432
mflodmancc3d4422017-08-03 08:27:51 -07002433 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002434}
2435
mflodmancc3d4422017-08-03 08:27:51 -07002436TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002437 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002439 DataRate::BitsPerSec(kTargetBitrateBps),
2440 DataRate::BitsPerSec(kTargetBitrateBps),
2441 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2442
2443 const int kWidth = 1280;
2444 const int kHeight = 720;
2445 int sequence = 1;
2446
2447 // Enable BALANCED preference, no initial limitation.
2448 test::FrameForwarder source;
2449 video_stream_encoder_->SetSource(&source,
2450 webrtc::DegradationPreference::BALANCED);
2451 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2452 WaitForEncodedFrame(sequence++);
2453 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2454 EXPECT_FALSE(stats.cpu_limited_resolution);
2455 EXPECT_FALSE(stats.cpu_limited_framerate);
2456 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2457
2458 // Trigger CPU overuse, should now adapt down.
2459 video_stream_encoder_->TriggerCpuOveruse();
2460 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2461 WaitForEncodedFrame(sequence++);
2462 stats = stats_proxy_->GetStats();
2463 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2464
2465 // Set new degradation preference should clear restrictions since we changed
2466 // from BALANCED.
2467 video_stream_encoder_->SetSource(
2468 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2469 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2470 WaitForEncodedFrame(sequence++);
2471 stats = stats_proxy_->GetStats();
2472 EXPECT_FALSE(stats.cpu_limited_resolution);
2473 EXPECT_FALSE(stats.cpu_limited_framerate);
2474 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2475
2476 // Force an input frame rate to be available, or the adaptation call won't
2477 // know what framerate to adapt from.
2478 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2479 mock_stats.input_frame_rate = 30;
2480 stats_proxy_->SetMockStats(mock_stats);
2481 video_stream_encoder_->TriggerCpuOveruse();
2482 stats_proxy_->ResetMockStats();
2483 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2484 WaitForEncodedFrame(sequence++);
2485
2486 // We have now adapted once.
2487 stats = stats_proxy_->GetStats();
2488 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2489
2490 // Back to BALANCED, should clear the restrictions again.
2491 video_stream_encoder_->SetSource(&source,
2492 webrtc::DegradationPreference::BALANCED);
2493 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2494 WaitForEncodedFrame(sequence++);
2495 stats = stats_proxy_->GetStats();
2496 EXPECT_FALSE(stats.cpu_limited_resolution);
2497 EXPECT_FALSE(stats.cpu_limited_framerate);
2498 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2499
2500 video_stream_encoder_->Stop();
2501}
2502
2503TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002504 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002506 DataRate::BitsPerSec(kTargetBitrateBps),
2507 DataRate::BitsPerSec(kTargetBitrateBps),
2508 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002509
asapersson0944a802017-04-07 00:57:58 -07002510 const int kWidth = 1280;
2511 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002512 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002513
asaperssonfab67072017-04-04 05:51:49 -07002514 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002515 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002516 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002517 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002518 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002519 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2520
asapersson02465b82017-04-10 01:12:52 -07002521 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002522 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002523 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002524 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002525 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002526 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002527 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002528 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2529
2530 // Set new source with adaptation still enabled.
2531 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002532 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002533 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002534
2535 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002536 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002537 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002538 stats = stats_proxy_->GetStats();
2539 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002540 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002541 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2542
sprangc5d62e22017-04-02 23:53:04 -07002543 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002544 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002545 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002546 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002547 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002548 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002549 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002550 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002551 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002552 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002553 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2554
sprangc5d62e22017-04-02 23:53:04 -07002555 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002556 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002557 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2558 mock_stats.input_frame_rate = 30;
2559 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002560 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002561 stats_proxy_->ResetMockStats();
2562
2563 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002564 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002565 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002566
2567 // Framerate now adapted.
2568 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002569 EXPECT_FALSE(stats.cpu_limited_resolution);
2570 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002571 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2572
2573 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002574 video_stream_encoder_->SetSource(&new_video_source,
2575 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002576 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002577 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002578 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002579
2580 stats = stats_proxy_->GetStats();
2581 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002582 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002583 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2584
2585 // Try to trigger overuse. Should not succeed.
2586 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002587 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002588 stats_proxy_->ResetMockStats();
2589
2590 stats = stats_proxy_->GetStats();
2591 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002592 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002593 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2594
2595 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002596 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002597 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002598 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002599 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002600 stats = stats_proxy_->GetStats();
2601 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002602 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002603 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002604
2605 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002606 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002607 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002608 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002609 stats = stats_proxy_->GetStats();
2610 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002611 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002612 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2613
2614 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002615 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002616 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002617 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002618 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002619 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002620 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002621 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002622 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002623 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002624 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2625
2626 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002627 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002628 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002629 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002630 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002631 stats = stats_proxy_->GetStats();
2632 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002633 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002634 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002635 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002636
mflodmancc3d4422017-08-03 08:27:51 -07002637 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002638}
2639
mflodmancc3d4422017-08-03 08:27:51 -07002640TEST_F(VideoStreamEncoderTest,
2641 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002642 const int kWidth = 1280;
2643 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002645 DataRate::BitsPerSec(kTargetBitrateBps),
2646 DataRate::BitsPerSec(kTargetBitrateBps),
2647 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002648
asaperssonfab67072017-04-04 05:51:49 -07002649 // Expect no scaling to begin with.
asapersson02465b82017-04-10 01:12:52 -07002650 VerifyNoLimitation(video_source_.sink_wants());
kthelgason876222f2016-11-29 01:44:11 -08002651
asaperssonfab67072017-04-04 05:51:49 -07002652 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002653 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002654
asaperssonfab67072017-04-04 05:51:49 -07002655 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002656 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002657
asaperssonfab67072017-04-04 05:51:49 -07002658 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002659 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002660
kthelgason876222f2016-11-29 01:44:11 -08002661 // Expect a scale down.
2662 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002663 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002664
asapersson02465b82017-04-10 01:12:52 -07002665 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002666 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002667 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002668 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002669
asaperssonfab67072017-04-04 05:51:49 -07002670 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002671 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002672 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002673 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002674
asaperssonfab67072017-04-04 05:51:49 -07002675 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002676 EXPECT_EQ(std::numeric_limits<int>::max(),
2677 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002678
asaperssonfab67072017-04-04 05:51:49 -07002679 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002680 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002681 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002682 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002683
asapersson02465b82017-04-10 01:12:52 -07002684 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002685 EXPECT_EQ(std::numeric_limits<int>::max(),
2686 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002687
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002689}
2690
mflodmancc3d4422017-08-03 08:27:51 -07002691TEST_F(VideoStreamEncoderTest,
2692 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002693 const int kWidth = 1280;
2694 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002696 DataRate::BitsPerSec(kTargetBitrateBps),
2697 DataRate::BitsPerSec(kTargetBitrateBps),
2698 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002699
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002700 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002701 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002702 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002703 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002704
2705 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002706 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002707 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002708 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2709 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2710
2711 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002712 video_stream_encoder_->TriggerCpuOveruse();
asapersson09f05612017-05-15 23:40:18 -07002713 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002714 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2715 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2716 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2717
2718 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002719 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002720 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2721 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2722 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2723
mflodmancc3d4422017-08-03 08:27:51 -07002724 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002725}
2726
mflodmancc3d4422017-08-03 08:27:51 -07002727TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002728 const int kWidth = 1280;
2729 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002730 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002731 DataRate::BitsPerSec(kTargetBitrateBps),
2732 DataRate::BitsPerSec(kTargetBitrateBps),
2733 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002734
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002735 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002736 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002737 video_stream_encoder_->SetSource(&source,
2738 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002739 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2740 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002741 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002742
2743 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002744 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002745 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
2746 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2747 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2748 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2749
2750 // Trigger adapt down for same input resolution, expect no change.
2751 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2752 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002753 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002754 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2755 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2756 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2757
2758 // Trigger adapt down for larger input resolution, expect no change.
2759 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2760 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002761 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002762 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2763 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2764 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2765
mflodmancc3d4422017-08-03 08:27:51 -07002766 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002767}
2768
mflodmancc3d4422017-08-03 08:27:51 -07002769TEST_F(VideoStreamEncoderTest,
2770 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002771 const int kWidth = 1280;
2772 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002774 DataRate::BitsPerSec(kTargetBitrateBps),
2775 DataRate::BitsPerSec(kTargetBitrateBps),
2776 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002777
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002778 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002779 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002780 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002781 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002782
2783 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002784 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002785 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002786 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2787 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2788
2789 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002790 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002791 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002792 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2793 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2794
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002796}
2797
mflodmancc3d4422017-08-03 08:27:51 -07002798TEST_F(VideoStreamEncoderTest,
2799 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002800 const int kWidth = 1280;
2801 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002803 DataRate::BitsPerSec(kTargetBitrateBps),
2804 DataRate::BitsPerSec(kTargetBitrateBps),
2805 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002806
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002807 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002808 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002809 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002810 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002811
2812 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002814 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002816 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2817
2818 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002819 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002820 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002821 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002822 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2823
mflodmancc3d4422017-08-03 08:27:51 -07002824 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002825}
2826
mflodmancc3d4422017-08-03 08:27:51 -07002827TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002828 const int kWidth = 1280;
2829 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002831 DataRate::BitsPerSec(kTargetBitrateBps),
2832 DataRate::BitsPerSec(kTargetBitrateBps),
2833 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002834
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002835 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002836 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002837 video_stream_encoder_->SetSource(&source,
2838 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002839
2840 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2841 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002842 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2846
2847 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002848 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002849 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07002850 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2851 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2852 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2853
mflodmancc3d4422017-08-03 08:27:51 -07002854 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002855}
2856
mflodmancc3d4422017-08-03 08:27:51 -07002857TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002858 const int kWidth = 1280;
2859 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002860 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002861 DataRate::BitsPerSec(kTargetBitrateBps),
2862 DataRate::BitsPerSec(kTargetBitrateBps),
2863 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002864
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002865 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002866 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002867 video_stream_encoder_->SetSource(&source,
2868 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002869
2870 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2871 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002872 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002873 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2874 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2875 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2876
2877 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002878 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002879 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002880 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2881 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2882 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2883
mflodmancc3d4422017-08-03 08:27:51 -07002884 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002885}
2886
mflodmancc3d4422017-08-03 08:27:51 -07002887TEST_F(VideoStreamEncoderTest,
2888 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002889 const int kWidth = 1280;
2890 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002892 DataRate::BitsPerSec(kTargetBitrateBps),
2893 DataRate::BitsPerSec(kTargetBitrateBps),
2894 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002895
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002896 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002897 AdaptingFrameForwarder source;
2898 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002899 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002900 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002901
2902 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002904 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002905 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2907
2908 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002910 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002911 WaitForEncodedFrame(2);
asapersson09f05612017-05-15 23:40:18 -07002912 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asapersson02465b82017-04-10 01:12:52 -07002913 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2914 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2915
2916 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002917 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002918 VerifyFpsMaxResolutionMax(source.sink_wants());
asapersson02465b82017-04-10 01:12:52 -07002919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2920 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2921 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2922
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002924}
2925
mflodmancc3d4422017-08-03 08:27:51 -07002926TEST_F(VideoStreamEncoderTest,
2927 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002928 const int kWidth = 1280;
2929 const int kHeight = 720;
2930 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002932 DataRate::BitsPerSec(kTargetBitrateBps),
2933 DataRate::BitsPerSec(kTargetBitrateBps),
2934 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002935
2936 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2937 stats.input_frame_rate = kInputFps;
2938 stats_proxy_->SetMockStats(stats);
2939
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002940 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002941 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2942 sink_.WaitForEncodedFrame(1);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002943 VerifyFpsMaxResolutionMax(video_source_.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002944
2945 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002946 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002947 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2948 sink_.WaitForEncodedFrame(2);
2949 VerifyFpsMaxResolutionLt(video_source_.sink_wants(), kWidth * kHeight);
2950
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002951 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002952 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002953 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002954 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002955 // Give the encoder queue time to process the change in degradation preference
2956 // by waiting for an encoded frame.
2957 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2958 sink_.WaitForEncodedFrame(3);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002959 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002960
2961 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002962 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002963 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2964 sink_.WaitForEncodedFrame(4);
asapersson09f05612017-05-15 23:40:18 -07002965 VerifyFpsLtResolutionMax(new_video_source.sink_wants(), kInputFps);
2966
2967 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002968 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002969 VerifyFpsMaxResolutionMax(new_video_source.sink_wants());
asapersson09f05612017-05-15 23:40:18 -07002970
mflodmancc3d4422017-08-03 08:27:51 -07002971 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002972}
2973
mflodmancc3d4422017-08-03 08:27:51 -07002974TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002975 const int kWidth = 1280;
2976 const int kHeight = 720;
2977 const size_t kNumFrames = 10;
2978
Henrik Boström381d1092020-05-12 18:49:07 +02002979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002980 DataRate::BitsPerSec(kTargetBitrateBps),
2981 DataRate::BitsPerSec(kTargetBitrateBps),
2982 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002983
asaperssond0de2952017-04-21 01:47:31 -07002984 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002985 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002986 video_source_.set_adaptation_enabled(true);
2987
2988 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2989 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2990
2991 int downscales = 0;
2992 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002993 video_source_.IncomingCapturedFrame(
2994 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2995 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002996
asaperssonfab67072017-04-04 05:51:49 -07002997 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002998 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002999 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003000 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003001
3002 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3003 ++downscales;
3004
3005 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3006 EXPECT_EQ(downscales,
3007 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3008 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003009 }
mflodmancc3d4422017-08-03 08:27:51 -07003010 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003011}
3012
mflodmancc3d4422017-08-03 08:27:51 -07003013TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003014 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3015 const int kWidth = 1280;
3016 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003018 DataRate::BitsPerSec(kTargetBitrateBps),
3019 DataRate::BitsPerSec(kTargetBitrateBps),
3020 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003021
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003022 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003023 AdaptingFrameForwarder source;
3024 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003025 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003026 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003027
Åsa Persson8c1bf952018-09-13 10:42:19 +02003028 int64_t timestamp_ms = kFrameIntervalMs;
3029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003030 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003031 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003032 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3033 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3034
3035 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003037 timestamp_ms += kFrameIntervalMs;
3038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3039 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003040 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003041 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3042 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3043
3044 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003045 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003046 timestamp_ms += kFrameIntervalMs;
3047 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003048 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003049 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003050 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3051 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3052
3053 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003054 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003055 timestamp_ms += kFrameIntervalMs;
3056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3057 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003058 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003059 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3060 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3061
3062 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003063 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003064 timestamp_ms += kFrameIntervalMs;
3065 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003066 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003067 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003068 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3069 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3070
mflodmancc3d4422017-08-03 08:27:51 -07003071 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003072}
3073
mflodmancc3d4422017-08-03 08:27:51 -07003074TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003075 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3076 const int kWidth = 1280;
3077 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003079 DataRate::BitsPerSec(kTargetBitrateBps),
3080 DataRate::BitsPerSec(kTargetBitrateBps),
3081 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003082
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003083 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003084 AdaptingFrameForwarder source;
3085 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003086 video_stream_encoder_->SetSource(&source,
3087 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003088
Åsa Persson8c1bf952018-09-13 10:42:19 +02003089 int64_t timestamp_ms = kFrameIntervalMs;
3090 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003091 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003092 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003093 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3094 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3095
3096 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003097 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003098 timestamp_ms += kFrameIntervalMs;
3099 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3100 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003101 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3102 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3103 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3104
3105 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003106 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003107 timestamp_ms += kFrameIntervalMs;
3108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003109 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003110 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003111 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3112 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3113
3114 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003116 timestamp_ms += kFrameIntervalMs;
3117 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3118 sink_.WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07003119 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
3120 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3121 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3122
3123 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003124 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003125 timestamp_ms += kFrameIntervalMs;
3126 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003127 sink_.WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003128 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07003129 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3130 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3131
mflodmancc3d4422017-08-03 08:27:51 -07003132 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003133}
3134
Sergey Silkin41c650b2019-10-14 13:12:19 +02003135TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3136 fake_encoder_.SetResolutionBitrateLimits(
3137 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3138
Henrik Boström381d1092020-05-12 18:49:07 +02003139 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003140 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3141 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3142 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3143 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003144
3145 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3146 AdaptingFrameForwarder source;
3147 source.set_adaptation_enabled(true);
3148 video_stream_encoder_->SetSource(
3149 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3150
3151 // Insert 720p frame.
3152 int64_t timestamp_ms = kFrameIntervalMs;
3153 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3154 WaitForEncodedFrame(1280, 720);
3155
3156 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003157 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003158 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3159 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3160 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3161 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003162 video_stream_encoder_->TriggerQualityLow();
3163
3164 // Insert 720p frame. It should be downscaled and encoded.
3165 timestamp_ms += kFrameIntervalMs;
3166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3167 WaitForEncodedFrame(960, 540);
3168
3169 // Trigger adapt up. Higher resolution should not be requested duo to lack
3170 // of bitrate.
3171 video_stream_encoder_->TriggerQualityHigh();
3172 VerifyFpsMaxResolutionLt(source.sink_wants(), 1280 * 720);
3173
3174 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003176 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3177 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3178 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3179 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003180
3181 // Trigger adapt up. Higher resolution should be requested.
3182 video_stream_encoder_->TriggerQualityHigh();
3183 VerifyFpsMaxResolutionMax(source.sink_wants());
3184
3185 video_stream_encoder_->Stop();
3186}
3187
3188TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3189 fake_encoder_.SetResolutionBitrateLimits(
3190 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3191
3192 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003194 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3195 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3196 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3197 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003198
3199 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3200 AdaptingFrameForwarder source;
3201 source.set_adaptation_enabled(true);
3202 video_stream_encoder_->SetSource(
3203 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3204
3205 // Insert 720p frame. It should be dropped and lower resolution should be
3206 // requested.
3207 int64_t timestamp_ms = kFrameIntervalMs;
3208 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3209 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003210 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003211
3212 // Insert 720p frame. It should be downscaled and encoded.
3213 timestamp_ms += kFrameIntervalMs;
3214 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3215 WaitForEncodedFrame(960, 540);
3216
3217 video_stream_encoder_->Stop();
3218}
3219
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003220class BalancedDegradationTest : public VideoStreamEncoderTest {
3221 protected:
3222 void SetupTest() {
3223 // Reset encoder for field trials to take effect.
3224 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003225 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003226
3227 // Enable BALANCED preference.
3228 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003229 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3230 }
3231
3232 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003233 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003234 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3235 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003236 }
3237
Åsa Persson45b176f2019-09-30 11:19:05 +02003238 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003239 timestamp_ms_ += kFrameIntervalMs;
3240 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003241 }
3242
3243 void InsertFrameAndWaitForEncoded() {
3244 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003245 sink_.WaitForEncodedFrame(timestamp_ms_);
3246 }
3247
3248 const int kWidth = 640; // pixels:640x360=230400
3249 const int kHeight = 360;
3250 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3251 int64_t timestamp_ms_ = 0;
3252 AdaptingFrameForwarder source_;
3253};
3254
3255TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3256 test::ScopedFieldTrials field_trials(
3257 "WebRTC-Video-BalancedDegradationSettings/"
3258 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3259 SetupTest();
3260
3261 // Force input frame rate.
3262 const int kInputFps = 24;
3263 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3264 stats.input_frame_rate = kInputFps;
3265 stats_proxy_->SetMockStats(stats);
3266
Åsa Persson45b176f2019-09-30 11:19:05 +02003267 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003268 VerifyFpsMaxResolutionMax(source_.sink_wants());
3269
3270 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003271 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3272 // QP samples.
3273 EXPECT_FALSE(
3274 video_stream_encoder_
3275 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003276 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3277
3278 video_stream_encoder_->Stop();
3279}
3280
3281TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3282 test::ScopedFieldTrials field_trials(
3283 "WebRTC-Video-BalancedDegradationSettings/"
3284 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3285 SetupTest();
3286
3287 // Force input frame rate.
3288 const int kInputFps = 25;
3289 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3290 stats.input_frame_rate = kInputFps;
3291 stats_proxy_->SetMockStats(stats);
3292
Åsa Persson45b176f2019-09-30 11:19:05 +02003293 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003294 VerifyFpsMaxResolutionMax(source_.sink_wants());
3295
3296 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003297 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3298 // samples.
3299 EXPECT_TRUE(
3300 video_stream_encoder_
3301 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003302 VerifyFpsEqResolutionMax(source_.sink_wants(), 24);
3303
3304 video_stream_encoder_->Stop();
3305}
3306
3307TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3308 test::ScopedFieldTrials field_trials(
3309 "WebRTC-Video-BalancedDegradationSettings/"
3310 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3311 SetupTest();
3312
3313 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3314
Åsa Persson45b176f2019-09-30 11:19:05 +02003315 InsertFrameAndWaitForEncoded();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003316 VerifyFpsMaxResolutionMax(source_.sink_wants());
3317
3318 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3319 video_stream_encoder_->TriggerQualityLow();
3320 VerifyFpsEqResolutionMax(source_.sink_wants(), 22);
3321
3322 video_stream_encoder_->Stop();
3323}
3324
Åsa Perssonccfb3402019-09-25 15:13:04 +02003325TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003326 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003327 "WebRTC-Video-BalancedDegradationSettings/"
3328 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003329 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003330
Åsa Persson1b247f12019-08-14 17:26:39 +02003331 const int kMinBitrateBps = 425000;
3332 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003333 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003334
Åsa Persson45b176f2019-09-30 11:19:05 +02003335 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003336 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003337 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3338
3339 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3340 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003341 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003342 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson1b247f12019-08-14 17:26:39 +02003343 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3344
3345 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3346 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003347 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003348 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003349 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3350
Åsa Persson30ab0152019-08-27 12:22:33 +02003351 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3352 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003353 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003354 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3355 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003356 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3357
3358 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003359 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003360 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003361 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003362
Åsa Persson30ab0152019-08-27 12:22:33 +02003363 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003364 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003365 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003366 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003367 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003368 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3369
3370 video_stream_encoder_->Stop();
3371}
3372
Åsa Perssonccfb3402019-09-25 15:13:04 +02003373TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003374 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3375 test::ScopedFieldTrials field_trials(
3376 "WebRTC-Video-BalancedDegradationSettings/"
3377 "pixels:57600|129600|230400,fps:7|24|24/");
3378 SetupTest();
3379 OnBitrateUpdated(kLowTargetBitrateBps);
3380
3381 VerifyNoLimitation(source_.sink_wants());
3382
3383 // Insert frame, expect scaled down:
3384 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3385 InsertFrame();
3386 EXPECT_FALSE(WaitForFrame(1000));
3387 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3388 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3389
3390 // Insert frame, expect scaled down:
3391 // resolution (320x180@24fps).
3392 InsertFrame();
3393 EXPECT_FALSE(WaitForFrame(1000));
3394 EXPECT_LT(source_.sink_wants().max_pixel_count,
3395 source_.last_wants().max_pixel_count);
3396 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3397
3398 // Frame should not be dropped (min pixels per frame reached).
3399 InsertFrameAndWaitForEncoded();
3400
3401 video_stream_encoder_->Stop();
3402}
3403
3404TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003405 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003406 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003407 "WebRTC-Video-BalancedDegradationSettings/"
3408 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003409 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003410
Åsa Persson30ab0152019-08-27 12:22:33 +02003411 const int kResolutionMinBitrateBps = 435000;
3412 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003413 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003414
Åsa Persson45b176f2019-09-30 11:19:05 +02003415 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003416 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003417 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3418
3419 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3420 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003421 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003422 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
3425 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3426 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003427 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003428 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003429 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430
3431 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3432 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003433 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003434 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003435 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
Åsa Persson30ab0152019-08-27 12:22:33 +02003437 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3438 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003439 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003440 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003441 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3442
3443 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3444 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003445 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003446 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3447
3448 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003449 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003450 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003451 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003452 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003453 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3454
3455 video_stream_encoder_->Stop();
3456}
3457
Åsa Perssonccfb3402019-09-25 15:13:04 +02003458TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003459 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003460 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003461 "WebRTC-Video-BalancedDegradationSettings/"
3462 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003463 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003464
Åsa Persson30ab0152019-08-27 12:22:33 +02003465 const int kMinBitrateBps = 425000;
3466 const int kTooLowMinBitrateBps = 424000;
3467 const int kResolutionMinBitrateBps = 435000;
3468 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003469 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003470
Åsa Persson45b176f2019-09-30 11:19:05 +02003471 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003472 VerifyFpsMaxResolutionMax(source_.sink_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003473 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3474
3475 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3476 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003477 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003478 VerifyFpsEqResolutionMax(source_.sink_wants(), 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003479 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3480
3481 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3482 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003483 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003484 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003485 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3486
3487 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3488 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003489 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003490 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003491 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492
3493 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3494 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003495 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003496 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3497
3498 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003499 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003500 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003501 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003502 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003503 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3504
3505 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003506 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003507 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003508 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003509 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3510
3511 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003512 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003513 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003514 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003515 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003516 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3517
Åsa Persson1b247f12019-08-14 17:26:39 +02003518 video_stream_encoder_->Stop();
3519}
3520
mflodmancc3d4422017-08-03 08:27:51 -07003521TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003522 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3523 const int kWidth = 1280;
3524 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003525 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003526 DataRate::BitsPerSec(kTargetBitrateBps),
3527 DataRate::BitsPerSec(kTargetBitrateBps),
3528 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003529
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003530 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003531 AdaptingFrameForwarder source;
3532 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003533 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003534 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003535
Åsa Persson8c1bf952018-09-13 10:42:19 +02003536 int64_t timestamp_ms = kFrameIntervalMs;
3537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003538 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003539 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003540 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3542 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3543 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3544
3545 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003546 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003547 timestamp_ms += kFrameIntervalMs;
3548 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3549 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003550 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
asaperssond0de2952017-04-21 01:47:31 -07003551 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3552 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3553 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3554 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3555
3556 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003557 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003558 timestamp_ms += kFrameIntervalMs;
3559 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3560 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003561 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003562 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3564 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3565 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3566
Jonathan Yubc771b72017-12-08 17:04:29 -08003567 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003568 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003569 timestamp_ms += kFrameIntervalMs;
3570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3571 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003572 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003573 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3574 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003575 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003576 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3577
Jonathan Yubc771b72017-12-08 17:04:29 -08003578 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003579 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003580 timestamp_ms += kFrameIntervalMs;
3581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3582 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003583 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003584 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003585 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3587 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3589
Jonathan Yubc771b72017-12-08 17:04:29 -08003590 // Trigger quality adapt down, expect no change (min resolution reached).
3591 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003592 timestamp_ms += kFrameIntervalMs;
3593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3594 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003595 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3596 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3598 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3599 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3600
3601 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003602 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003603 timestamp_ms += kFrameIntervalMs;
3604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3605 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003606 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003607 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3609 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3610 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3611
3612 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003613 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003614 timestamp_ms += kFrameIntervalMs;
3615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3616 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003617 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3618 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3620 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3621 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3622
3623 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003624 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003625 timestamp_ms += kFrameIntervalMs;
3626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3627 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003628 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003629 last_wants = source.sink_wants();
3630 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003632 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003633 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3634
3635 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003636 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003637 timestamp_ms += kFrameIntervalMs;
3638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3639 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003640 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003641 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003643 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003644 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3645
3646 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003647 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003648 timestamp_ms += kFrameIntervalMs;
3649 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003650 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003651 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02003652 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07003653 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003655 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003656 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003657
mflodmancc3d4422017-08-03 08:27:51 -07003658 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003659}
3660
mflodmancc3d4422017-08-03 08:27:51 -07003661TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003662 const int kWidth = 640;
3663 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003664
Henrik Boström381d1092020-05-12 18:49:07 +02003665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003666 DataRate::BitsPerSec(kTargetBitrateBps),
3667 DataRate::BitsPerSec(kTargetBitrateBps),
3668 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003669
perkj803d97f2016-11-01 11:45:46 -07003670 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003671 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003672 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003673 }
3674
mflodmancc3d4422017-08-03 08:27:51 -07003675 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003676 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003677 video_source_.IncomingCapturedFrame(CreateFrame(
3678 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003679 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003680 }
3681
mflodmancc3d4422017-08-03 08:27:51 -07003682 video_stream_encoder_->Stop();
3683 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003684 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003685
Ying Wangef3998f2019-12-09 13:06:53 +01003686 EXPECT_METRIC_EQ(
3687 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3688 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003689 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3690}
3691
mflodmancc3d4422017-08-03 08:27:51 -07003692TEST_F(VideoStreamEncoderTest,
3693 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003695 DataRate::BitsPerSec(kTargetBitrateBps),
3696 DataRate::BitsPerSec(kTargetBitrateBps),
3697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003698 const int kWidth = 640;
3699 const int kHeight = 360;
3700
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003701 video_stream_encoder_->SetSource(&video_source_,
3702 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003703
3704 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3705 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003706 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003707 }
3708
mflodmancc3d4422017-08-03 08:27:51 -07003709 video_stream_encoder_->Stop();
3710 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003711 stats_proxy_.reset();
3712
3713 EXPECT_EQ(0,
3714 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3715}
3716
mflodmancc3d4422017-08-03 08:27:51 -07003717TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003718 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003719 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003720
3721 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003722 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003723 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003724 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3725 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003726
sprang57c2fff2017-01-16 06:24:02 -08003727 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003728 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003729 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003730 DataRate::BitsPerSec(kLowTargetBitrateBps),
3731 DataRate::BitsPerSec(kLowTargetBitrateBps),
3732 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003733
sprang57c2fff2017-01-16 06:24:02 -08003734 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003735 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3736 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003737 VideoBitrateAllocation bitrate_allocation =
3738 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003739 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003740 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003741 // TODO(srte): The use of millisecs here looks like an error, but the tests
3742 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003743 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003744
3745 // Not called on second frame.
3746 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3747 .Times(0);
3748 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003749 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3750 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003751 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003752
3753 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003754 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3755 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003756 const int64_t start_time_ms = rtc::TimeMillis();
3757 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3758 video_source_.IncomingCapturedFrame(
3759 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3760 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003761 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003762 }
3763
3764 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003765 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003766
mflodmancc3d4422017-08-03 08:27:51 -07003767 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003768}
3769
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003770TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3771 // 2 TLs configured, temporal layers supported by encoder.
3772 const int kNumTemporalLayers = 2;
3773 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3774 fake_encoder_.SetTemporalLayersSupported(0, true);
3775
3776 // Bitrate allocated across temporal layers.
3777 const int kTl0Bps = kTargetBitrateBps *
3778 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003779 kNumTemporalLayers, /*temporal_id*/ 0,
3780 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003781 const int kTl1Bps = kTargetBitrateBps *
3782 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003783 kNumTemporalLayers, /*temporal_id*/ 1,
3784 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003785 VideoBitrateAllocation expected_bitrate;
3786 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3787 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3788
3789 VerifyAllocatedBitrate(expected_bitrate);
3790 video_stream_encoder_->Stop();
3791}
3792
3793TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3794 // 2 TLs configured, temporal layers not supported by encoder.
3795 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3796 fake_encoder_.SetTemporalLayersSupported(0, false);
3797
3798 // Temporal layers not supported by the encoder.
3799 // Total bitrate should be at ti:0.
3800 VideoBitrateAllocation expected_bitrate;
3801 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3802
3803 VerifyAllocatedBitrate(expected_bitrate);
3804 video_stream_encoder_->Stop();
3805}
3806
3807TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3808 // 2 TLs configured, temporal layers only supported for first stream.
3809 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3810 fake_encoder_.SetTemporalLayersSupported(0, true);
3811 fake_encoder_.SetTemporalLayersSupported(1, false);
3812
3813 const int kS0Bps = 150000;
3814 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003815 kS0Bps *
3816 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3817 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003818 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003819 kS0Bps *
3820 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3821 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003822 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3823 // Temporal layers not supported by si:1.
3824 VideoBitrateAllocation expected_bitrate;
3825 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3826 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3827 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3828
3829 VerifyAllocatedBitrate(expected_bitrate);
3830 video_stream_encoder_->Stop();
3831}
3832
Niels Möller7dc26b72017-12-06 10:27:48 +01003833TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3834 const int kFrameWidth = 1280;
3835 const int kFrameHeight = 720;
3836 const int kFramerate = 24;
3837
Henrik Boström381d1092020-05-12 18:49:07 +02003838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003839 DataRate::BitsPerSec(kTargetBitrateBps),
3840 DataRate::BitsPerSec(kTargetBitrateBps),
3841 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003842 test::FrameForwarder source;
3843 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003844 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003845
3846 // Insert a single frame, triggering initial configuration.
3847 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3848 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3849
3850 EXPECT_EQ(
3851 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3852 kDefaultFramerate);
3853
3854 // Trigger reconfigure encoder (without resetting the entire instance).
3855 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003856 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003857 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3858 video_encoder_config.number_of_streams = 1;
3859 video_encoder_config.video_stream_factory =
3860 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3861 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003862 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003863 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3864
3865 // Detector should be updated with fps limit from codec config.
3866 EXPECT_EQ(
3867 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3868 kFramerate);
3869
3870 // Trigger overuse, max framerate should be reduced.
3871 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3872 stats.input_frame_rate = kFramerate;
3873 stats_proxy_->SetMockStats(stats);
3874 video_stream_encoder_->TriggerCpuOveruse();
3875 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3876 int adapted_framerate =
3877 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3878 EXPECT_LT(adapted_framerate, kFramerate);
3879
3880 // Trigger underuse, max framerate should go back to codec configured fps.
3881 // Set extra low fps, to make sure it's actually reset, not just incremented.
3882 stats = stats_proxy_->GetStats();
3883 stats.input_frame_rate = adapted_framerate / 2;
3884 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003885 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003886 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3887 EXPECT_EQ(
3888 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3889 kFramerate);
3890
3891 video_stream_encoder_->Stop();
3892}
3893
3894TEST_F(VideoStreamEncoderTest,
3895 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3896 const int kFrameWidth = 1280;
3897 const int kFrameHeight = 720;
3898 const int kLowFramerate = 15;
3899 const int kHighFramerate = 25;
3900
Henrik Boström381d1092020-05-12 18:49:07 +02003901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003902 DataRate::BitsPerSec(kTargetBitrateBps),
3903 DataRate::BitsPerSec(kTargetBitrateBps),
3904 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003905 test::FrameForwarder source;
3906 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003907 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003908
3909 // Trigger initial configuration.
3910 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003911 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003912 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3913 video_encoder_config.number_of_streams = 1;
3914 video_encoder_config.video_stream_factory =
3915 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3916 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3917 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003918 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003919 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3920
3921 EXPECT_EQ(
3922 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3923 kLowFramerate);
3924
3925 // Trigger overuse, max framerate should be reduced.
3926 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3927 stats.input_frame_rate = kLowFramerate;
3928 stats_proxy_->SetMockStats(stats);
3929 video_stream_encoder_->TriggerCpuOveruse();
3930 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3931 int adapted_framerate =
3932 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3933 EXPECT_LT(adapted_framerate, kLowFramerate);
3934
3935 // Reconfigure the encoder with a new (higher max framerate), max fps should
3936 // still respect the adaptation.
3937 video_encoder_config.video_stream_factory =
3938 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3939 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3940 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003941 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003942 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3943
3944 EXPECT_EQ(
3945 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3946 adapted_framerate);
3947
3948 // Trigger underuse, max framerate should go back to codec configured fps.
3949 stats = stats_proxy_->GetStats();
3950 stats.input_frame_rate = adapted_framerate;
3951 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003952 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003953 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3954 EXPECT_EQ(
3955 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3956 kHighFramerate);
3957
3958 video_stream_encoder_->Stop();
3959}
3960
mflodmancc3d4422017-08-03 08:27:51 -07003961TEST_F(VideoStreamEncoderTest,
3962 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003963 const int kFrameWidth = 1280;
3964 const int kFrameHeight = 720;
3965 const int kFramerate = 24;
3966
Henrik Boström381d1092020-05-12 18:49:07 +02003967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003968 DataRate::BitsPerSec(kTargetBitrateBps),
3969 DataRate::BitsPerSec(kTargetBitrateBps),
3970 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003971 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003972 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003973 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003974
3975 // Trigger initial configuration.
3976 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003977 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003978 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3979 video_encoder_config.number_of_streams = 1;
3980 video_encoder_config.video_stream_factory =
3981 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3982 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003983 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003984 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003985 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003986
Niels Möller7dc26b72017-12-06 10:27:48 +01003987 EXPECT_EQ(
3988 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3989 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003990
3991 // Trigger overuse, max framerate should be reduced.
3992 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3993 stats.input_frame_rate = kFramerate;
3994 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003995 video_stream_encoder_->TriggerCpuOveruse();
3996 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003997 int adapted_framerate =
3998 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003999 EXPECT_LT(adapted_framerate, kFramerate);
4000
4001 // Change degradation preference to not enable framerate scaling. Target
4002 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004003 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004004 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004005 EXPECT_EQ(
4006 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4007 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004008
mflodmancc3d4422017-08-03 08:27:51 -07004009 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004010}
4011
mflodmancc3d4422017-08-03 08:27:51 -07004012TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004013 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004014 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004015 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4016 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4017 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004018 const int kWidth = 640;
4019 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004020
asaperssonfab67072017-04-04 05:51:49 -07004021 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004022
4023 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004024 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004025
4026 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004027 EXPECT_TRUE_WAIT(
4028 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004029
sprangc5d62e22017-04-02 23:53:04 -07004030 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004031
asaperssonfab67072017-04-04 05:51:49 -07004032 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004033 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004034 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004035
4036 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004037 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004038
Henrik Boström2671dac2020-05-19 16:29:09 +02004039 EXPECT_TRUE_WAIT(
4040 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004041
mflodmancc3d4422017-08-03 08:27:51 -07004042 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004043}
4044
mflodmancc3d4422017-08-03 08:27:51 -07004045TEST_F(VideoStreamEncoderTest,
4046 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004047 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004049 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4050 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4051 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004052 const int kWidth = 640;
4053 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004054
4055 // We expect the n initial frames to get dropped.
4056 int i;
4057 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004058 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004059 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004060 }
4061 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004062 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004063 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004064
4065 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004066 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004067
mflodmancc3d4422017-08-03 08:27:51 -07004068 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004069}
4070
mflodmancc3d4422017-08-03 08:27:51 -07004071TEST_F(VideoStreamEncoderTest,
4072 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004073 const int kWidth = 640;
4074 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004075 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004076 DataRate::BitsPerSec(kLowTargetBitrateBps),
4077 DataRate::BitsPerSec(kLowTargetBitrateBps),
4078 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004079
4080 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004081 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004082 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004083
asaperssonfab67072017-04-04 05:51:49 -07004084 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004085 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004086 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004087
mflodmancc3d4422017-08-03 08:27:51 -07004088 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004089}
4090
mflodmancc3d4422017-08-03 08:27:51 -07004091TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004092 const int kWidth = 640;
4093 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004094 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004095
4096 VideoEncoderConfig video_encoder_config;
4097 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4098 // Make format different, to force recreation of encoder.
4099 video_encoder_config.video_format.parameters["foo"] = "foo";
4100 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004101 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004102 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004103 DataRate::BitsPerSec(kLowTargetBitrateBps),
4104 DataRate::BitsPerSec(kLowTargetBitrateBps),
4105 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004106
kthelgasonb83797b2017-02-14 11:57:25 -08004107 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004108 video_stream_encoder_->SetSource(&video_source_,
4109 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004110
asaperssonfab67072017-04-04 05:51:49 -07004111 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004112 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004113 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004114
mflodmancc3d4422017-08-03 08:27:51 -07004115 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004116 fake_encoder_.SetQualityScaling(true);
4117}
4118
Åsa Persson139f4dc2019-08-02 09:29:58 +02004119TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4120 webrtc::test::ScopedFieldTrials field_trials(
4121 "WebRTC-Video-QualityScalerSettings/"
4122 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4123 // Reset encoder for field trials to take effect.
4124 ConfigureEncoder(video_encoder_config_.Copy());
4125 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4126 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4127 const int kWidth = 640;
4128 const int kHeight = 360;
4129
Henrik Boström381d1092020-05-12 18:49:07 +02004130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004131 DataRate::BitsPerSec(kTargetBitrateBps),
4132 DataRate::BitsPerSec(kTargetBitrateBps),
4133 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004134 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4135 // Frame should not be dropped.
4136 WaitForEncodedFrame(1);
4137
Henrik Boström381d1092020-05-12 18:49:07 +02004138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004139 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4140 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4141 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004142 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4143 // Frame should not be dropped.
4144 WaitForEncodedFrame(2);
4145
Henrik Boström381d1092020-05-12 18:49:07 +02004146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004147 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4148 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4149 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004150 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4151 // Expect to drop this frame, the wait should time out.
4152 ExpectDroppedFrame();
4153
4154 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004155 EXPECT_TRUE_WAIT(
4156 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004157 video_stream_encoder_->Stop();
4158}
4159
Åsa Perssone644a032019-11-08 15:56:00 +01004160TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4161 webrtc::test::ScopedFieldTrials field_trials(
4162 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4163
4164 // Reset encoder for field trials to take effect.
4165 VideoEncoderConfig config = video_encoder_config_.Copy();
4166 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004167 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004168 ConfigureEncoder(std::move(config));
4169 fake_encoder_.SetQp(kQpLow);
4170
4171 // Enable MAINTAIN_FRAMERATE preference.
4172 AdaptingFrameForwarder source;
4173 source.set_adaptation_enabled(true);
4174 video_stream_encoder_->SetSource(&source,
4175 DegradationPreference::MAINTAIN_FRAMERATE);
4176
4177 // Start at low bitrate.
4178 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4180 DataRate::BitsPerSec(kLowBitrateBps),
4181 DataRate::BitsPerSec(kLowBitrateBps),
4182 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004183
4184 // Expect first frame to be dropped and resolution to be limited.
4185 const int kWidth = 1280;
4186 const int kHeight = 720;
4187 const int64_t kFrameIntervalMs = 100;
4188 int64_t timestamp_ms = kFrameIntervalMs;
4189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4190 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004191 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4192 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004193
4194 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4196 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004197
4198 // Insert frames and advance |min_duration_ms|.
4199 for (size_t i = 1; i <= 10; i++) {
4200 timestamp_ms += kFrameIntervalMs;
4201 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4202 WaitForEncodedFrame(timestamp_ms);
4203 }
4204 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4205 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4206
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004207 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004208
4209 // Insert frame should trigger high BW and release quality limitation.
4210 timestamp_ms += kFrameIntervalMs;
4211 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4212 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004213 // The ramp-up code involves the adaptation queue, give it time to execute.
4214 // TODO(hbos): Can we await an appropriate event instead?
4215 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Åsa Perssone644a032019-11-08 15:56:00 +01004216 VerifyFpsMaxResolutionMax(source.sink_wants());
4217
4218 // Frame should not be adapted.
4219 timestamp_ms += kFrameIntervalMs;
4220 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4221 WaitForEncodedFrame(kWidth, kHeight);
4222 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4223
4224 video_stream_encoder_->Stop();
4225}
4226
mflodmancc3d4422017-08-03 08:27:51 -07004227TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004228 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4229 const int kTooSmallWidth = 10;
4230 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004232 DataRate::BitsPerSec(kTargetBitrateBps),
4233 DataRate::BitsPerSec(kTargetBitrateBps),
4234 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004235
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004236 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004237 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004238 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004239 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004240 VerifyNoLimitation(source.sink_wants());
4241 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4242
4243 // Trigger adapt down, too small frame, expect no change.
4244 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004245 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004246 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004247 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssond0de2952017-04-21 01:47:31 -07004248 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4249 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4250
mflodmancc3d4422017-08-03 08:27:51 -07004251 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004252}
4253
mflodmancc3d4422017-08-03 08:27:51 -07004254TEST_F(VideoStreamEncoderTest,
4255 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004256 const int kTooSmallWidth = 10;
4257 const int kTooSmallHeight = 10;
4258 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004259 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004260 DataRate::BitsPerSec(kTargetBitrateBps),
4261 DataRate::BitsPerSec(kTargetBitrateBps),
4262 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004263
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004264 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004265 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004266 video_stream_encoder_->SetSource(&source,
4267 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004268 VerifyNoLimitation(source.sink_wants());
4269 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4271
4272 // Trigger adapt down, expect limited framerate.
4273 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004274 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004275 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004276 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4277 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4278 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4279 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4280
4281 // Trigger adapt down, too small frame, expect no change.
4282 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004283 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004284 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004285 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4286 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4287 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4288 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4289
mflodmancc3d4422017-08-03 08:27:51 -07004290 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004291}
4292
mflodmancc3d4422017-08-03 08:27:51 -07004293TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004294 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004295 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004296 DataRate::BitsPerSec(kTargetBitrateBps),
4297 DataRate::BitsPerSec(kTargetBitrateBps),
4298 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004299 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004300 const int kFrameWidth = 1280;
4301 const int kFrameHeight = 720;
4302 video_source_.IncomingCapturedFrame(
4303 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004304 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004305 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004306}
4307
sprangb1ca0732017-02-01 08:38:12 -08004308// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004309TEST_F(VideoStreamEncoderTest,
4310 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004312 DataRate::BitsPerSec(kTargetBitrateBps),
4313 DataRate::BitsPerSec(kTargetBitrateBps),
4314 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004315
4316 const int kFrameWidth = 1280;
4317 const int kFrameHeight = 720;
4318 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004319 // requested by
4320 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004321 video_source_.set_adaptation_enabled(true);
4322
4323 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004324 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004325 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004326
4327 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004328 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004329 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004330 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004331 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004332
asaperssonfab67072017-04-04 05:51:49 -07004333 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004334 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004335 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004336 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004337 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004338
mflodmancc3d4422017-08-03 08:27:51 -07004339 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004340}
sprangfe627f32017-03-29 08:24:59 -07004341
mflodmancc3d4422017-08-03 08:27:51 -07004342TEST_F(VideoStreamEncoderTest,
4343 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004344 const int kFrameWidth = 1280;
4345 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004346
Henrik Boström381d1092020-05-12 18:49:07 +02004347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004348 DataRate::BitsPerSec(kTargetBitrateBps),
4349 DataRate::BitsPerSec(kTargetBitrateBps),
4350 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004351 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004352 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004353 video_source_.set_adaptation_enabled(true);
4354
sprang4847ae62017-06-27 07:06:52 -07004355 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004356
4357 video_source_.IncomingCapturedFrame(
4358 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004359 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004360
4361 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004362 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004363
4364 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004365 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004366 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004367 video_source_.IncomingCapturedFrame(
4368 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004369 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004370 }
4371
4372 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004373 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004374 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004375 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004376 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004377 video_source_.IncomingCapturedFrame(
4378 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004379 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004380 ++num_frames_dropped;
4381 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004382 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004383 }
4384 }
4385
sprang4847ae62017-06-27 07:06:52 -07004386 // Add some slack to account for frames dropped by the frame dropper.
4387 const int kErrorMargin = 1;
4388 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004389 kErrorMargin);
4390
4391 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004392 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004393 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004394 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004395 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004396 video_source_.IncomingCapturedFrame(
4397 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004398 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004399 ++num_frames_dropped;
4400 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004401 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004402 }
4403 }
sprang4847ae62017-06-27 07:06:52 -07004404 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004405 kErrorMargin);
4406
4407 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004408 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004409 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004410 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004411 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004412 video_source_.IncomingCapturedFrame(
4413 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004414 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004415 ++num_frames_dropped;
4416 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004417 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004418 }
4419 }
sprang4847ae62017-06-27 07:06:52 -07004420 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004421 kErrorMargin);
4422
4423 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004424 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004425 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004426 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004427 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004428 video_source_.IncomingCapturedFrame(
4429 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004430 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004431 ++num_frames_dropped;
4432 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004433 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004434 }
4435 }
4436 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4437
mflodmancc3d4422017-08-03 08:27:51 -07004438 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004439}
4440
mflodmancc3d4422017-08-03 08:27:51 -07004441TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004442 const int kFramerateFps = 5;
4443 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004444 const int kFrameWidth = 1280;
4445 const int kFrameHeight = 720;
4446
sprang4847ae62017-06-27 07:06:52 -07004447 // Reconfigure encoder with two temporal layers and screensharing, which will
4448 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004449 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004450
Henrik Boström381d1092020-05-12 18:49:07 +02004451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004452 DataRate::BitsPerSec(kTargetBitrateBps),
4453 DataRate::BitsPerSec(kTargetBitrateBps),
4454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004455 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004456 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004457 video_source_.set_adaptation_enabled(true);
4458
sprang4847ae62017-06-27 07:06:52 -07004459 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004460
4461 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004462 rtc::VideoSinkWants last_wants;
4463 do {
4464 last_wants = video_source_.sink_wants();
4465
sprangc5d62e22017-04-02 23:53:04 -07004466 // Insert frames to get a new fps estimate...
4467 for (int j = 0; j < kFramerateFps; ++j) {
4468 video_source_.IncomingCapturedFrame(
4469 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004470 if (video_source_.last_sent_width()) {
4471 sink_.WaitForEncodedFrame(timestamp_ms);
4472 }
sprangc5d62e22017-04-02 23:53:04 -07004473 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004474 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004475 }
4476 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004477 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004478 } while (video_source_.sink_wants().max_framerate_fps <
4479 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004480
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kMinFramerateFps);
asaperssonf7e294d2017-06-13 23:25:22 -07004482
mflodmancc3d4422017-08-03 08:27:51 -07004483 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004484}
asaperssonf7e294d2017-06-13 23:25:22 -07004485
mflodmancc3d4422017-08-03 08:27:51 -07004486TEST_F(VideoStreamEncoderTest,
4487 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004488 const int kWidth = 1280;
4489 const int kHeight = 720;
4490 const int64_t kFrameIntervalMs = 150;
4491 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004492 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004493 DataRate::BitsPerSec(kTargetBitrateBps),
4494 DataRate::BitsPerSec(kTargetBitrateBps),
4495 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004496
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004497 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004498 AdaptingFrameForwarder source;
4499 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004500 video_stream_encoder_->SetSource(&source,
4501 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004502 timestamp_ms += kFrameIntervalMs;
4503 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004504 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004505 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004506 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4507 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4508 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4509
4510 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004511 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004512 timestamp_ms += kFrameIntervalMs;
4513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004514 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004515 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4517 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4518 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4519
4520 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004521 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004522 timestamp_ms += kFrameIntervalMs;
4523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004524 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004525 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4528 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4529
4530 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004531 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004532 timestamp_ms += kFrameIntervalMs;
4533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004534 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004535 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4538 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4539
4540 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004542 timestamp_ms += kFrameIntervalMs;
4543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004544 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004545 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4548 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4549
4550 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004551 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004552 timestamp_ms += kFrameIntervalMs;
4553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004554 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004555 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4558 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4559
4560 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004562 timestamp_ms += kFrameIntervalMs;
4563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004564 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004565 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4568 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4569
4570 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004571 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004572 timestamp_ms += kFrameIntervalMs;
4573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004574 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004575 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4576 rtc::VideoSinkWants last_wants = source.sink_wants();
4577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4579 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4580
4581 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004582 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004583 timestamp_ms += kFrameIntervalMs;
4584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004585 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004586 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4589 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4590
4591 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004592 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004593 timestamp_ms += kFrameIntervalMs;
4594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004595 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004596 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4599 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4600
4601 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004602 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004603 timestamp_ms += kFrameIntervalMs;
4604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004605 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004606 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4609 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4610
4611 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004612 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004613 timestamp_ms += kFrameIntervalMs;
4614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004615 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004616 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4619 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4620
4621 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004622 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004623 timestamp_ms += kFrameIntervalMs;
4624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004625 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004626 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4629 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4630
4631 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004632 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004633 timestamp_ms += kFrameIntervalMs;
4634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004635 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004636 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4638 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4639 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4640
4641 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004642 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004643 timestamp_ms += kFrameIntervalMs;
4644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004645 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004646 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4649 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4650
Åsa Persson30ab0152019-08-27 12:22:33 +02004651 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004652 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004653 timestamp_ms += kFrameIntervalMs;
4654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004655 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004656 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004657 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4659 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4660 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4661
4662 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004663 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004664 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004665 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4666
mflodmancc3d4422017-08-03 08:27:51 -07004667 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004668}
4669
mflodmancc3d4422017-08-03 08:27:51 -07004670TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004671 const int kWidth = 1280;
4672 const int kHeight = 720;
4673 const int64_t kFrameIntervalMs = 150;
4674 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004676 DataRate::BitsPerSec(kTargetBitrateBps),
4677 DataRate::BitsPerSec(kTargetBitrateBps),
4678 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004679
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004680 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004681 AdaptingFrameForwarder source;
4682 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004683 video_stream_encoder_->SetSource(&source,
4684 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004685 timestamp_ms += kFrameIntervalMs;
4686 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004687 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004688 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4690 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4691 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4692 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4693 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4694 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4695
4696 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004697 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004698 timestamp_ms += kFrameIntervalMs;
4699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004700 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004701 VerifyFpsMaxResolutionLt(source.sink_wants(), kWidth * kHeight);
4702 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4703 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4704 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4705 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4706 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4708
4709 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004710 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004711 timestamp_ms += kFrameIntervalMs;
4712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004713 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004714 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4717 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4719 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4721
4722 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004723 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004724 timestamp_ms += kFrameIntervalMs;
4725 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004726 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004727 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4728 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4729 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4730 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4732 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4733 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4734
4735 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004736 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004737 timestamp_ms += kFrameIntervalMs;
4738 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004739 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004740 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4741 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4743 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4745 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4746 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4747
4748 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004749 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004750 timestamp_ms += kFrameIntervalMs;
4751 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004752 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004753 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4754 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4755 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4756 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4757 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4758 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4759 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4760
4761 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004762 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004763 timestamp_ms += kFrameIntervalMs;
4764 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004765 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004766 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Åsa Persson8c1bf952018-09-13 10:42:19 +02004767 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004768 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4769 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4770 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4771 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4772 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4773 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4774
4775 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004776 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004777 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004778 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4779 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4780
mflodmancc3d4422017-08-03 08:27:51 -07004781 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004782}
4783
mflodmancc3d4422017-08-03 08:27:51 -07004784TEST_F(VideoStreamEncoderTest,
4785 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004786 const int kWidth = 640;
4787 const int kHeight = 360;
4788 const int kFpsLimit = 15;
4789 const int64_t kFrameIntervalMs = 150;
4790 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004791 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004792 DataRate::BitsPerSec(kTargetBitrateBps),
4793 DataRate::BitsPerSec(kTargetBitrateBps),
4794 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004795
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004796 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004797 AdaptingFrameForwarder source;
4798 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004799 video_stream_encoder_->SetSource(&source,
4800 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004801 timestamp_ms += kFrameIntervalMs;
4802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004803 WaitForEncodedFrame(kWidth, kHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004804 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004805 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4807 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4809 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4810 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4811
4812 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004813 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004814 timestamp_ms += kFrameIntervalMs;
4815 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004816 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004817 VerifyFpsEqResolutionMax(source.sink_wants(), kFpsLimit);
4818 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4819 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4820 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4821 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4822 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4823 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4824
4825 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004826 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004827 timestamp_ms += kFrameIntervalMs;
4828 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004829 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004830 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4831 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4832 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4833 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4834 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4835 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4836 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4837
4838 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004839 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004840 timestamp_ms += kFrameIntervalMs;
4841 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004842 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004843 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4844 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4845 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4846 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4847 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4848 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4849 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4850
4851 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004852 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004853 timestamp_ms += kFrameIntervalMs;
4854 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004855 WaitForEncodedFrame(timestamp_ms);
Åsa Persson8c1bf952018-09-13 10:42:19 +02004856 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004857 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4858 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4859 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4860 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4861 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4862 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4863
4864 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004865 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004866 VerifyFpsMaxResolutionMax(source.sink_wants());
asaperssonf7e294d2017-06-13 23:25:22 -07004867 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4868 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4869
mflodmancc3d4422017-08-03 08:27:51 -07004870 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004871}
4872
mflodmancc3d4422017-08-03 08:27:51 -07004873TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004874 const int kFrameWidth = 1920;
4875 const int kFrameHeight = 1080;
4876 // 3/4 of 1920.
4877 const int kAdaptedFrameWidth = 1440;
4878 // 3/4 of 1080 rounded down to multiple of 4.
4879 const int kAdaptedFrameHeight = 808;
4880 const int kFramerate = 24;
4881
Henrik Boström381d1092020-05-12 18:49:07 +02004882 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004883 DataRate::BitsPerSec(kTargetBitrateBps),
4884 DataRate::BitsPerSec(kTargetBitrateBps),
4885 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004886 // Trigger reconfigure encoder (without resetting the entire instance).
4887 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004888 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004889 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4890 video_encoder_config.number_of_streams = 1;
4891 video_encoder_config.video_stream_factory =
4892 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004893 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004894 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004895 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004896
4897 video_source_.set_adaptation_enabled(true);
4898
4899 video_source_.IncomingCapturedFrame(
4900 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004901 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004902
4903 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004904 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004905 video_source_.IncomingCapturedFrame(
4906 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004907 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004908
mflodmancc3d4422017-08-03 08:27:51 -07004909 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004910}
4911
mflodmancc3d4422017-08-03 08:27:51 -07004912TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004913 const int kFrameWidth = 1280;
4914 const int kFrameHeight = 720;
4915 const int kLowFps = 2;
4916 const int kHighFps = 30;
4917
Henrik Boström381d1092020-05-12 18:49:07 +02004918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004919 DataRate::BitsPerSec(kTargetBitrateBps),
4920 DataRate::BitsPerSec(kTargetBitrateBps),
4921 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004922
4923 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4924 max_framerate_ = kLowFps;
4925
4926 // Insert 2 seconds of 2fps video.
4927 for (int i = 0; i < kLowFps * 2; ++i) {
4928 video_source_.IncomingCapturedFrame(
4929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4930 WaitForEncodedFrame(timestamp_ms);
4931 timestamp_ms += 1000 / kLowFps;
4932 }
4933
4934 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004936 DataRate::BitsPerSec(kTargetBitrateBps),
4937 DataRate::BitsPerSec(kTargetBitrateBps),
4938 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004939 video_source_.IncomingCapturedFrame(
4940 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4941 WaitForEncodedFrame(timestamp_ms);
4942 timestamp_ms += 1000 / kLowFps;
4943
4944 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4945
4946 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004947 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004948 const int kFrameIntervalMs = 1000 / kHighFps;
4949 max_framerate_ = kHighFps;
4950 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4951 video_source_.IncomingCapturedFrame(
4952 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4953 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4954 // be dropped if the encoder hans't been updated with the new higher target
4955 // framerate yet, causing it to overshoot the target bitrate and then
4956 // suffering the wrath of the media optimizer.
4957 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4958 timestamp_ms += kFrameIntervalMs;
4959 }
4960
4961 // Don expect correct measurement just yet, but it should be higher than
4962 // before.
4963 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4964
mflodmancc3d4422017-08-03 08:27:51 -07004965 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004966}
4967
mflodmancc3d4422017-08-03 08:27:51 -07004968TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004969 const int kFrameWidth = 1280;
4970 const int kFrameHeight = 720;
4971 const int kTargetBitrateBps = 1000000;
4972
4973 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004974 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02004975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004976 DataRate::BitsPerSec(kTargetBitrateBps),
4977 DataRate::BitsPerSec(kTargetBitrateBps),
4978 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004979 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004980
4981 // Insert a first video frame, causes another bitrate update.
4982 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4983 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4984 video_source_.IncomingCapturedFrame(
4985 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4986 WaitForEncodedFrame(timestamp_ms);
4987
4988 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02004989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4990 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
4991 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07004992
4993 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02004994 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004995 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07004996
4997 // Bitrate observer should not be called.
4998 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
4999 video_source_.IncomingCapturedFrame(
5000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5001 ExpectDroppedFrame();
5002
mflodmancc3d4422017-08-03 08:27:51 -07005003 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005004}
ilnik6b826ef2017-06-16 06:53:48 -07005005
Niels Möller4db138e2018-04-19 09:04:13 +02005006TEST_F(VideoStreamEncoderTest,
5007 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5008 const int kFrameWidth = 1280;
5009 const int kFrameHeight = 720;
5010 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005011 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005012 DataRate::BitsPerSec(kTargetBitrateBps),
5013 DataRate::BitsPerSec(kTargetBitrateBps),
5014 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005015 video_source_.IncomingCapturedFrame(
5016 CreateFrame(1, kFrameWidth, kFrameHeight));
5017 WaitForEncodedFrame(1);
5018 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5019 .low_encode_usage_threshold_percent,
5020 default_options.low_encode_usage_threshold_percent);
5021 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5022 .high_encode_usage_threshold_percent,
5023 default_options.high_encode_usage_threshold_percent);
5024 video_stream_encoder_->Stop();
5025}
5026
5027TEST_F(VideoStreamEncoderTest,
5028 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5029 const int kFrameWidth = 1280;
5030 const int kFrameHeight = 720;
5031 CpuOveruseOptions hardware_options;
5032 hardware_options.low_encode_usage_threshold_percent = 150;
5033 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005034 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005035
Henrik Boström381d1092020-05-12 18:49:07 +02005036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005037 DataRate::BitsPerSec(kTargetBitrateBps),
5038 DataRate::BitsPerSec(kTargetBitrateBps),
5039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005040 video_source_.IncomingCapturedFrame(
5041 CreateFrame(1, kFrameWidth, kFrameHeight));
5042 WaitForEncodedFrame(1);
5043 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5044 .low_encode_usage_threshold_percent,
5045 hardware_options.low_encode_usage_threshold_percent);
5046 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5047 .high_encode_usage_threshold_percent,
5048 hardware_options.high_encode_usage_threshold_percent);
5049 video_stream_encoder_->Stop();
5050}
5051
Niels Möller6bb5ab92019-01-11 11:11:10 +01005052TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5053 const int kFrameWidth = 320;
5054 const int kFrameHeight = 240;
5055 const int kFps = 30;
5056 const int kTargetBitrateBps = 120000;
5057 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5058
Henrik Boström381d1092020-05-12 18:49:07 +02005059 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005060 DataRate::BitsPerSec(kTargetBitrateBps),
5061 DataRate::BitsPerSec(kTargetBitrateBps),
5062 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005063
5064 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5065 max_framerate_ = kFps;
5066
5067 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5068 fake_encoder_.SimulateOvershoot(1.0);
5069 int num_dropped = 0;
5070 for (int i = 0; i < kNumFramesInRun; ++i) {
5071 video_source_.IncomingCapturedFrame(
5072 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5073 // Wait up to two frame durations for a frame to arrive.
5074 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5075 ++num_dropped;
5076 }
5077 timestamp_ms += 1000 / kFps;
5078 }
5079
Erik Språnga8d48ab2019-02-08 14:17:40 +01005080 // Framerate should be measured to be near the expected target rate.
5081 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5082
5083 // Frame drops should be within 5% of expected 0%.
5084 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005085
5086 // Make encoder produce frames at double the expected bitrate during 3 seconds
5087 // of video, verify number of drops. Rate needs to be slightly changed in
5088 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005089 double overshoot_factor = 2.0;
5090 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5091 // With bitrate adjuster, when need to overshoot even more to trigger
5092 // frame dropping.
5093 overshoot_factor *= 2;
5094 }
5095 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005096 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005097 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5098 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5099 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005100 num_dropped = 0;
5101 for (int i = 0; i < kNumFramesInRun; ++i) {
5102 video_source_.IncomingCapturedFrame(
5103 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5104 // Wait up to two frame durations for a frame to arrive.
5105 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5106 ++num_dropped;
5107 }
5108 timestamp_ms += 1000 / kFps;
5109 }
5110
Henrik Boström381d1092020-05-12 18:49:07 +02005111 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005112 DataRate::BitsPerSec(kTargetBitrateBps),
5113 DataRate::BitsPerSec(kTargetBitrateBps),
5114 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005115
5116 // Target framerate should be still be near the expected target, despite
5117 // the frame drops.
5118 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5119
5120 // Frame drops should be within 5% of expected 50%.
5121 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005122
5123 video_stream_encoder_->Stop();
5124}
5125
5126TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5127 const int kFrameWidth = 320;
5128 const int kFrameHeight = 240;
5129 const int kActualInputFps = 24;
5130 const int kTargetBitrateBps = 120000;
5131
5132 ASSERT_GT(max_framerate_, kActualInputFps);
5133
5134 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5135 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005137 DataRate::BitsPerSec(kTargetBitrateBps),
5138 DataRate::BitsPerSec(kTargetBitrateBps),
5139 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005140
5141 // Insert 3 seconds of video, with an input fps lower than configured max.
5142 for (int i = 0; i < kActualInputFps * 3; ++i) {
5143 video_source_.IncomingCapturedFrame(
5144 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5145 // Wait up to two frame durations for a frame to arrive.
5146 WaitForEncodedFrame(timestamp_ms);
5147 timestamp_ms += 1000 / kActualInputFps;
5148 }
5149
5150 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5151
5152 video_stream_encoder_->Stop();
5153}
5154
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005155TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5156 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005157 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005158 DataRate::BitsPerSec(kTargetBitrateBps),
5159 DataRate::BitsPerSec(kTargetBitrateBps),
5160 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005161
5162 fake_encoder_.BlockNextEncode();
5163 video_source_.IncomingCapturedFrame(
5164 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5165 WaitForEncodedFrame(1);
5166 // On the very first frame full update should be forced.
5167 rect = fake_encoder_.GetLastUpdateRect();
5168 EXPECT_EQ(rect.offset_x, 0);
5169 EXPECT_EQ(rect.offset_y, 0);
5170 EXPECT_EQ(rect.height, codec_height_);
5171 EXPECT_EQ(rect.width, codec_width_);
5172 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5173 // call to ContinueEncode.
5174 video_source_.IncomingCapturedFrame(
5175 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5176 ExpectDroppedFrame();
5177 video_source_.IncomingCapturedFrame(
5178 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5179 ExpectDroppedFrame();
5180 fake_encoder_.ContinueEncode();
5181 WaitForEncodedFrame(3);
5182 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5183 rect = fake_encoder_.GetLastUpdateRect();
5184 EXPECT_EQ(rect.offset_x, 1);
5185 EXPECT_EQ(rect.offset_y, 0);
5186 EXPECT_EQ(rect.width, 10);
5187 EXPECT_EQ(rect.height, 1);
5188
5189 video_source_.IncomingCapturedFrame(
5190 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5191 WaitForEncodedFrame(4);
5192 // Previous frame was encoded, so no accumulation should happen.
5193 rect = fake_encoder_.GetLastUpdateRect();
5194 EXPECT_EQ(rect.offset_x, 0);
5195 EXPECT_EQ(rect.offset_y, 0);
5196 EXPECT_EQ(rect.width, 1);
5197 EXPECT_EQ(rect.height, 1);
5198
5199 video_stream_encoder_->Stop();
5200}
5201
Erik Språngd7329ca2019-02-21 21:19:53 +01005202TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005203 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005204 DataRate::BitsPerSec(kTargetBitrateBps),
5205 DataRate::BitsPerSec(kTargetBitrateBps),
5206 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005207
5208 // First frame is always keyframe.
5209 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5210 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005211 EXPECT_THAT(
5212 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005213 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005214
5215 // Insert delta frame.
5216 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5217 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005218 EXPECT_THAT(
5219 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005220 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005221
5222 // Request next frame be a key-frame.
5223 video_stream_encoder_->SendKeyFrame();
5224 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5225 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005226 EXPECT_THAT(
5227 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005228 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005229
5230 video_stream_encoder_->Stop();
5231}
5232
5233TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5234 // Setup simulcast with three streams.
5235 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005236 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005237 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5238 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5239 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005240 // Wait for all three layers before triggering event.
5241 sink_.SetNumExpectedLayers(3);
5242
5243 // First frame is always keyframe.
5244 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5245 WaitForEncodedFrame(1);
5246 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005247 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5248 VideoFrameType::kVideoFrameKey,
5249 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005250
5251 // Insert delta frame.
5252 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5253 WaitForEncodedFrame(2);
5254 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005255 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5256 VideoFrameType::kVideoFrameDelta,
5257 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005258
5259 // Request next frame be a key-frame.
5260 // Only first stream is configured to produce key-frame.
5261 video_stream_encoder_->SendKeyFrame();
5262 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5263 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005264
5265 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5266 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005267 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005268 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005269 VideoFrameType::kVideoFrameKey,
5270 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005271
5272 video_stream_encoder_->Stop();
5273}
5274
5275TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5276 // Configure internal source factory and setup test again.
5277 encoder_factory_.SetHasInternalSource(true);
5278 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005280 DataRate::BitsPerSec(kTargetBitrateBps),
5281 DataRate::BitsPerSec(kTargetBitrateBps),
5282 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005283
5284 // Call encoder directly, simulating internal source where encoded frame
5285 // callback in VideoStreamEncoder is called despite no OnFrame().
5286 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5287 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005288 EXPECT_THAT(
5289 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005290 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005291
Niels Möller8f7ce222019-03-21 15:43:58 +01005292 const std::vector<VideoFrameType> kDeltaFrame = {
5293 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005294 // Need to set timestamp manually since manually for injected frame.
5295 VideoFrame frame = CreateFrame(101, nullptr);
5296 frame.set_timestamp(101);
5297 fake_encoder_.InjectFrame(frame, false);
5298 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005299 EXPECT_THAT(
5300 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005301 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005302
5303 // Request key-frame. The forces a dummy frame down into the encoder.
5304 fake_encoder_.ExpectNullFrame();
5305 video_stream_encoder_->SendKeyFrame();
5306 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005307 EXPECT_THAT(
5308 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005309 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005310
5311 video_stream_encoder_->Stop();
5312}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005313
5314TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5315 // Configure internal source factory and setup test again.
5316 encoder_factory_.SetHasInternalSource(true);
5317 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005319 DataRate::BitsPerSec(kTargetBitrateBps),
5320 DataRate::BitsPerSec(kTargetBitrateBps),
5321 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005322
5323 int64_t timestamp = 1;
5324 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005325 image.SetEncodedData(
5326 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005327 image.capture_time_ms_ = ++timestamp;
5328 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5329 const int64_t kEncodeFinishDelayMs = 10;
5330 image.timing_.encode_start_ms = timestamp;
5331 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5332 fake_encoder_.InjectEncodedImage(image);
5333 // Wait for frame without incrementing clock.
5334 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5335 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5336 // capture timestamp should be kEncodeFinishDelayMs in the past.
5337 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5338 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5339 kEncodeFinishDelayMs);
5340
5341 video_stream_encoder_->Stop();
5342}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005343
5344TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5345 // Configure internal source factory and setup test again.
5346 encoder_factory_.SetHasInternalSource(true);
5347 ResetEncoder("H264", 1, 1, 1, false);
5348
5349 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5350 image._frameType = VideoFrameType::kVideoFrameKey;
5351
5352 CodecSpecificInfo codec_specific_info;
5353 codec_specific_info.codecType = kVideoCodecH264;
5354
5355 RTPFragmentationHeader fragmentation;
5356 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5357 fragmentation.fragmentationOffset[0] = 4;
5358 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5359
5360 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5361 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5362
5363 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5364 testing::ElementsAreArray(optimal_sps));
5365 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5366 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5367 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5368 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5369
5370 video_stream_encoder_->Stop();
5371}
5372
5373TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5374 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5375 0x00, 0x00, 0x03, 0x03, 0xF4,
5376 0x05, 0x03, 0xC7, 0xC0};
5377
5378 // Configure internal source factory and setup test again.
5379 encoder_factory_.SetHasInternalSource(true);
5380 ResetEncoder("H264", 1, 1, 1, false);
5381
5382 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5383 image._frameType = VideoFrameType::kVideoFrameKey;
5384
5385 CodecSpecificInfo codec_specific_info;
5386 codec_specific_info.codecType = kVideoCodecH264;
5387
5388 RTPFragmentationHeader fragmentation;
5389 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5390 fragmentation.fragmentationOffset[0] = 4;
5391 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5392
5393 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5394 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5395
5396 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5397 testing::ElementsAreArray(optimal_sps));
5398 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5399 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5400 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5401 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5402
5403 video_stream_encoder_->Stop();
5404}
5405
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005406TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5407 const int kFrameWidth = 1280;
5408 const int kFrameHeight = 720;
5409 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5410
Henrik Boström381d1092020-05-12 18:49:07 +02005411 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005412 DataRate::BitsPerSec(kTargetBitrateBps),
5413 DataRate::BitsPerSec(kTargetBitrateBps),
5414 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005415 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5416
5417 // Insert a first video frame. It should be dropped because of downscale in
5418 // resolution.
5419 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5420 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5421 frame.set_rotation(kVideoRotation_270);
5422 video_source_.IncomingCapturedFrame(frame);
5423
5424 ExpectDroppedFrame();
5425
5426 // Second frame is downscaled.
5427 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5428 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5429 frame.set_rotation(kVideoRotation_90);
5430 video_source_.IncomingCapturedFrame(frame);
5431
5432 WaitForEncodedFrame(timestamp_ms);
5433 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5434
5435 // Insert another frame, also downscaled.
5436 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5437 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5438 frame.set_rotation(kVideoRotation_180);
5439 video_source_.IncomingCapturedFrame(frame);
5440
5441 WaitForEncodedFrame(timestamp_ms);
5442 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5443
5444 video_stream_encoder_->Stop();
5445}
5446
Erik Språng5056af02019-09-02 15:53:11 +02005447TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5448 const int kFrameWidth = 320;
5449 const int kFrameHeight = 180;
5450
5451 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005453 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5454 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5455 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005456 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005457 /*rtt_ms=*/0,
5458 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005459
5460 // Insert a first video frame so that encoder gets configured.
5461 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5462 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5463 frame.set_rotation(kVideoRotation_270);
5464 video_source_.IncomingCapturedFrame(frame);
5465 WaitForEncodedFrame(timestamp_ms);
5466
5467 // Set a target rate below the minimum allowed by the codec settings.
5468 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005469 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5470 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005472 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005473 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005474 /*link_allocation=*/target_rate,
5475 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005476 /*rtt_ms=*/0,
5477 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005478 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5479
5480 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5481 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5482 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005483 DataRate allocation_sum =
5484 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005485 EXPECT_EQ(min_rate, allocation_sum);
5486 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5487
5488 video_stream_encoder_->Stop();
5489}
5490
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005491TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005492 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005493 DataRate::BitsPerSec(kTargetBitrateBps),
5494 DataRate::BitsPerSec(kTargetBitrateBps),
5495 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005496 // Capture a frame and wait for it to synchronize with the encoder thread.
5497 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5498 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5499 WaitForEncodedFrame(1);
5500
5501 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5502 ASSERT_TRUE(prev_rate_settings.has_value());
5503 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5504 kDefaultFramerate);
5505
5506 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5507 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5508 timestamp_ms += 1000 / kDefaultFramerate;
5509 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5510 WaitForEncodedFrame(timestamp_ms);
5511 }
5512 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5513 kDefaultFramerate);
5514 // Capture larger frame to trigger a reconfigure.
5515 codec_height_ *= 2;
5516 codec_width_ *= 2;
5517 timestamp_ms += 1000 / kDefaultFramerate;
5518 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5519 WaitForEncodedFrame(timestamp_ms);
5520
5521 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5522 auto current_rate_settings =
5523 fake_encoder_.GetAndResetLastRateControlSettings();
5524 // Ensure we have actually reconfigured twice
5525 // The rate settings should have been set again even though
5526 // they haven't changed.
5527 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005528 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005529
5530 video_stream_encoder_->Stop();
5531}
5532
philipeld9cc8c02019-09-16 14:53:40 +02005533struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005534 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5535 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5536 MOCK_METHOD(void,
5537 RequestEncoderSwitch,
5538 (const webrtc::SdpVideoFormat& format),
5539 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005540};
5541
5542TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5543 constexpr int kDontCare = 100;
5544
5545 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5546 video_send_config_.encoder_settings.encoder_switch_request_callback =
5547 &switch_callback;
5548 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5549 encoder_config.codec_type = kVideoCodecVP8;
5550 webrtc::test::ScopedFieldTrials field_trial(
5551 "WebRTC-NetworkCondition-EncoderSwitch/"
5552 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5553 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5554
5555 // Reset encoder for new configuration to take effect.
5556 ConfigureEncoder(std::move(encoder_config));
5557
5558 // Send one frame to trigger ReconfigureEncoder.
5559 video_source_.IncomingCapturedFrame(
5560 CreateFrame(kDontCare, kDontCare, kDontCare));
5561
5562 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005563 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5564 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005565 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005566 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005567
Henrik Boström381d1092020-05-12 18:49:07 +02005568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005569 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5570 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5571 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005572 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005573 /*rtt_ms=*/0,
5574 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005575
5576 video_stream_encoder_->Stop();
5577}
5578
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005579TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5580 constexpr int kDontCare = 100;
5581
5582 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5583 video_send_config_.encoder_settings.encoder_switch_request_callback =
5584 &switch_callback;
5585 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5586 encoder_config.codec_type = kVideoCodecVP8;
5587 webrtc::test::ScopedFieldTrials field_trial(
5588 "WebRTC-NetworkCondition-EncoderSwitch/"
5589 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5590 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5591
5592 // Reset encoder for new configuration to take effect.
5593 ConfigureEncoder(std::move(encoder_config));
5594
5595 // Send one frame to trigger ReconfigureEncoder.
5596 video_source_.IncomingCapturedFrame(
5597 CreateFrame(kDontCare, kDontCare, kDontCare));
5598
5599 using Config = EncoderSwitchRequestCallback::Config;
5600 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5601 .Times(0);
5602
Henrik Boström381d1092020-05-12 18:49:07 +02005603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005604 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5605 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5606 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5607 /*fraction_lost=*/0,
5608 /*rtt_ms=*/0,
5609 /*cwnd_reduce_ratio=*/0);
5610
5611 video_stream_encoder_->Stop();
5612}
5613
philipeld9cc8c02019-09-16 14:53:40 +02005614TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5615 constexpr int kSufficientBitrateToNotDrop = 1000;
5616 constexpr int kHighRes = 500;
5617 constexpr int kLowRes = 100;
5618
5619 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5620 video_send_config_.encoder_settings.encoder_switch_request_callback =
5621 &switch_callback;
5622 webrtc::test::ScopedFieldTrials field_trial(
5623 "WebRTC-NetworkCondition-EncoderSwitch/"
5624 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5625 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5626 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5627 encoder_config.codec_type = kVideoCodecH264;
5628
5629 // Reset encoder for new configuration to take effect.
5630 ConfigureEncoder(std::move(encoder_config));
5631
5632 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5633 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5634 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005636 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5637 /*stable_target_bitrate=*/
5638 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5639 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005640 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005641 /*rtt_ms=*/0,
5642 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005643
5644 // Send one frame to trigger ReconfigureEncoder.
5645 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5646 WaitForEncodedFrame(1);
5647
5648 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005649 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5650 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005651 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005652 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005653
5654 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5655 WaitForEncodedFrame(2);
5656
5657 video_stream_encoder_->Stop();
5658}
5659
philipel9b058032020-02-10 11:30:00 +01005660TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5661 constexpr int kDontCare = 100;
5662 StrictMock<MockEncoderSelector> encoder_selector;
5663 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5664 &fake_encoder_, &encoder_selector);
5665 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5666
5667 // Reset encoder for new configuration to take effect.
5668 ConfigureEncoder(video_encoder_config_.Copy());
5669
5670 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5671
5672 video_source_.IncomingCapturedFrame(
5673 CreateFrame(kDontCare, kDontCare, kDontCare));
5674 video_stream_encoder_->Stop();
5675
5676 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5677 // to it's factory, so in order for the encoder instance in the
5678 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5679 // reset the |video_stream_encoder_| here.
5680 video_stream_encoder_.reset();
5681}
5682
5683TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5684 constexpr int kDontCare = 100;
5685
5686 NiceMock<MockEncoderSelector> encoder_selector;
5687 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5688 video_send_config_.encoder_settings.encoder_switch_request_callback =
5689 &switch_callback;
5690 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5691 &fake_encoder_, &encoder_selector);
5692 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5693
5694 // Reset encoder for new configuration to take effect.
5695 ConfigureEncoder(video_encoder_config_.Copy());
5696
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005697 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005698 .WillByDefault(Return(SdpVideoFormat("AV1")));
5699 EXPECT_CALL(switch_callback,
5700 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5701 Field(&SdpVideoFormat::name, "AV1"))));
5702
Henrik Boström381d1092020-05-12 18:49:07 +02005703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005704 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5705 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5706 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005707 /*fraction_lost=*/0,
5708 /*rtt_ms=*/0,
5709 /*cwnd_reduce_ratio=*/0);
5710
5711 video_stream_encoder_->Stop();
5712}
5713
5714TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5715 constexpr int kSufficientBitrateToNotDrop = 1000;
5716 constexpr int kDontCare = 100;
5717
5718 NiceMock<MockVideoEncoder> video_encoder;
5719 NiceMock<MockEncoderSelector> encoder_selector;
5720 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5721 video_send_config_.encoder_settings.encoder_switch_request_callback =
5722 &switch_callback;
5723 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5724 &video_encoder, &encoder_selector);
5725 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5726
5727 // Reset encoder for new configuration to take effect.
5728 ConfigureEncoder(video_encoder_config_.Copy());
5729
5730 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5731 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5732 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005734 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5735 /*stable_target_bitrate=*/
5736 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5737 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005738 /*fraction_lost=*/0,
5739 /*rtt_ms=*/0,
5740 /*cwnd_reduce_ratio=*/0);
5741
5742 ON_CALL(video_encoder, Encode(_, _))
5743 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5744 ON_CALL(encoder_selector, OnEncoderBroken())
5745 .WillByDefault(Return(SdpVideoFormat("AV2")));
5746
5747 rtc::Event encode_attempted;
5748 EXPECT_CALL(switch_callback,
5749 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5750 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5751 EXPECT_EQ(format.name, "AV2");
5752 encode_attempted.Set();
5753 });
5754
5755 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5756 encode_attempted.Wait(3000);
5757
5758 video_stream_encoder_->Stop();
5759
5760 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5761 // to it's factory, so in order for the encoder instance in the
5762 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5763 // reset the |video_stream_encoder_| here.
5764 video_stream_encoder_.reset();
5765}
5766
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005767TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005768 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005769 const int kFrameWidth = 320;
5770 const int kFrameHeight = 180;
5771
5772 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005773 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005775 /*target_bitrate=*/rate,
5776 /*stable_target_bitrate=*/rate,
5777 /*link_allocation=*/rate,
5778 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005779 /*rtt_ms=*/0,
5780 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005781
5782 // Insert a first video frame so that encoder gets configured.
5783 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5784 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5785 frame.set_rotation(kVideoRotation_270);
5786 video_source_.IncomingCapturedFrame(frame);
5787 WaitForEncodedFrame(timestamp_ms);
5788 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5789
5790 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005791 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005793 /*target_bitrate=*/new_stable_rate,
5794 /*stable_target_bitrate=*/new_stable_rate,
5795 /*link_allocation=*/rate,
5796 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005797 /*rtt_ms=*/0,
5798 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005799 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5800 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5801 video_stream_encoder_->Stop();
5802}
5803
5804TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005805 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005806 const int kFrameWidth = 320;
5807 const int kFrameHeight = 180;
5808
5809 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005810 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005812 /*target_bitrate=*/rate,
5813 /*stable_target_bitrate=*/rate,
5814 /*link_allocation=*/rate,
5815 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005816 /*rtt_ms=*/0,
5817 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005818
5819 // Insert a first video frame so that encoder gets configured.
5820 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5821 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5822 frame.set_rotation(kVideoRotation_270);
5823 video_source_.IncomingCapturedFrame(frame);
5824 WaitForEncodedFrame(timestamp_ms);
5825 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5826
5827 // Set a higher target rate without changing the link_allocation. Should not
5828 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005829 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005831 /*target_bitrate=*/rate,
5832 /*stable_target_bitrate=*/new_stable_rate,
5833 /*link_allocation=*/rate,
5834 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005835 /*rtt_ms=*/0,
5836 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005837 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5838 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5839 video_stream_encoder_->Stop();
5840}
5841
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005842TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5843 test::ScopedFieldTrials field_trials(
5844 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5845 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5846 const int kFramerateFps = 30;
5847 const int kWidth = 1920;
5848 const int kHeight = 1080;
5849 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5850 // Works on screenshare mode.
5851 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5852 // We rely on the automatic resolution adaptation, but we handle framerate
5853 // adaptation manually by mocking the stats proxy.
5854 video_source_.set_adaptation_enabled(true);
5855
5856 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005858 DataRate::BitsPerSec(kTargetBitrateBps),
5859 DataRate::BitsPerSec(kTargetBitrateBps),
5860 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005861 video_stream_encoder_->SetSource(&video_source_,
5862 webrtc::DegradationPreference::BALANCED);
5863 VerifyNoLimitation(video_source_.sink_wants());
5864
5865 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5866 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5867
5868 // Pass enough frames with the full update to trigger animation detection.
5869 for (int i = 0; i < kNumFrames; ++i) {
5870 int64_t timestamp_ms =
5871 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5872 frame.set_ntp_time_ms(timestamp_ms);
5873 frame.set_timestamp_us(timestamp_ms * 1000);
5874 video_source_.IncomingCapturedFrame(frame);
5875 WaitForEncodedFrame(timestamp_ms);
5876 }
5877
5878 // Resolution should be limited.
5879 rtc::VideoSinkWants expected;
5880 expected.max_framerate_fps = kFramerateFps;
5881 expected.max_pixel_count = 1280 * 720 + 1;
5882 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5883
5884 // Pass one frame with no known update.
5885 // Resolution cap should be removed immediately.
5886 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5887 frame.set_ntp_time_ms(timestamp_ms);
5888 frame.set_timestamp_us(timestamp_ms * 1000);
5889 frame.clear_update_rect();
5890
5891 video_source_.IncomingCapturedFrame(frame);
5892 WaitForEncodedFrame(timestamp_ms);
5893
5894 // Resolution should be unlimited now.
5895 VerifyFpsEqResolutionMax(video_source_.sink_wants(), kFramerateFps);
5896
5897 video_stream_encoder_->Stop();
5898}
5899
perkj26091b12016-09-01 01:17:40 -07005900} // namespace webrtc