blob: 0bb7257ab9175b646946120043c446fe3dd61551 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020037#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080039#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010040#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020041#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "system_wrappers/include/sleep.h"
43#include "test/encoder_settings.h"
44#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020045#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010046#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020047#include "test/gmock.h"
48#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020049#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070051
52namespace webrtc {
53
sprang57c2fff2017-01-16 06:24:02 -080054using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020055using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020056using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020057using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020058using ::testing::Ge;
59using ::testing::Gt;
60using ::testing::Le;
61using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010062using ::testing::Matcher;
63using ::testing::NiceMock;
64using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020065using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080066
perkj803d97f2016-11-01 11:45:46 -070067namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020068const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010069const int kQpLow = 1;
70const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020071const int kMinFramerateFps = 2;
72const int kMinBalancedFramerateFps = 7;
73const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080074const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010075const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020076const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010077const uint32_t kSimulcastTargetBitrateBps = 3150000;
78const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080079const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070080const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020081const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020082const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020083const VideoEncoder::ResolutionBitrateLimits
84 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
85const VideoEncoder::ResolutionBitrateLimits
86 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080087
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020088uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
89 0x00, 0x00, 0x03, 0x03, 0xF4,
90 0x05, 0x03, 0xC7, 0xE0, 0x1B,
91 0x41, 0x10, 0x8D, 0x00};
92
perkj803d97f2016-11-01 11:45:46 -070093class TestBuffer : public webrtc::I420Buffer {
94 public:
95 TestBuffer(rtc::Event* event, int width, int height)
96 : I420Buffer(width, height), event_(event) {}
97
98 private:
99 friend class rtc::RefCountedObject<TestBuffer>;
100 ~TestBuffer() override {
101 if (event_)
102 event_->Set();
103 }
104 rtc::Event* const event_;
105};
106
Noah Richards51db4212019-06-12 06:59:12 -0700107// A fake native buffer that can't be converted to I420.
108class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
109 public:
110 FakeNativeBuffer(rtc::Event* event, int width, int height)
111 : event_(event), width_(width), height_(height) {}
112 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
113 int width() const override { return width_; }
114 int height() const override { return height_; }
115 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
116 return nullptr;
117 }
118
119 private:
120 friend class rtc::RefCountedObject<FakeNativeBuffer>;
121 ~FakeNativeBuffer() override {
122 if (event_)
123 event_->Set();
124 }
125 rtc::Event* const event_;
126 const int width_;
127 const int height_;
128};
129
Niels Möller7dc26b72017-12-06 10:27:48 +0100130class CpuOveruseDetectorProxy : public OveruseFrameDetector {
131 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200132 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
133 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200134 last_target_framerate_fps_(-1),
135 framerate_updated_event_(true /* manual_reset */,
136 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100137 virtual ~CpuOveruseDetectorProxy() {}
138
139 void OnTargetFramerateUpdated(int framerate_fps) override {
140 rtc::CritScope cs(&lock_);
141 last_target_framerate_fps_ = framerate_fps;
142 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200143 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 }
145
146 int GetLastTargetFramerate() {
147 rtc::CritScope cs(&lock_);
148 return last_target_framerate_fps_;
149 }
150
Niels Möller4db138e2018-04-19 09:04:13 +0200151 CpuOveruseOptions GetOptions() { return options_; }
152
Henrik Boström381d1092020-05-12 18:49:07 +0200153 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
154
Niels Möller7dc26b72017-12-06 10:27:48 +0100155 private:
156 rtc::CriticalSection lock_;
157 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200158 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100159};
160
Henrik Boström91aa7322020-04-28 12:24:33 +0200161class FakeQualityScalerQpUsageHandlerCallback
162 : public QualityScalerQpUsageHandlerCallbackInterface {
163 public:
164 FakeQualityScalerQpUsageHandlerCallback()
Henrik Boström5bf60e42020-05-13 15:20:25 +0200165 : QualityScalerQpUsageHandlerCallbackInterface(),
166 qp_usage_handled_event_(/*manual_reset=*/true,
167 /*initially_signaled=*/false),
168 clear_qp_samples_result_(absl::nullopt) {}
169 ~FakeQualityScalerQpUsageHandlerCallback() override {
170 RTC_DCHECK(clear_qp_samples_result_.has_value());
171 }
Henrik Boström91aa7322020-04-28 12:24:33 +0200172
173 void OnQpUsageHandled(bool clear_qp_samples) override {
174 clear_qp_samples_result_ = clear_qp_samples;
Henrik Boström5bf60e42020-05-13 15:20:25 +0200175 qp_usage_handled_event_.Set();
Henrik Boström91aa7322020-04-28 12:24:33 +0200176 }
177
Henrik Boström5bf60e42020-05-13 15:20:25 +0200178 bool WaitForQpUsageHandled() { return qp_usage_handled_event_.Wait(5000); }
179
Henrik Boström91aa7322020-04-28 12:24:33 +0200180 absl::optional<bool> clear_qp_samples_result() const {
181 return clear_qp_samples_result_;
182 }
183
184 private:
Henrik Boström5bf60e42020-05-13 15:20:25 +0200185 rtc::Event qp_usage_handled_event_;
Henrik Boström91aa7322020-04-28 12:24:33 +0200186 absl::optional<bool> clear_qp_samples_result_;
187};
188
Henrik Boström381d1092020-05-12 18:49:07 +0200189class VideoSourceRestrictionsUpdatedListener
190 : public ResourceAdaptationProcessorListener {
191 public:
192 VideoSourceRestrictionsUpdatedListener()
193 : was_restrictions_updated_(false), restrictions_updated_event_() {}
194 ~VideoSourceRestrictionsUpdatedListener() override {
195 RTC_DCHECK(was_restrictions_updated_);
196 }
197
198 rtc::Event* restrictions_updated_event() {
199 return &restrictions_updated_event_;
200 }
201
202 // ResourceAdaptationProcessorListener implementation.
203 void OnVideoSourceRestrictionsUpdated(
204 VideoSourceRestrictions restrictions,
205 const VideoAdaptationCounters& adaptation_counters,
206 rtc::scoped_refptr<Resource> reason) override {
207 was_restrictions_updated_ = true;
208 restrictions_updated_event_.Set();
209 }
210
211 private:
212 bool was_restrictions_updated_;
213 rtc::Event restrictions_updated_event_;
214};
215
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200216auto ResolutionMax() {
217 return AllOf(
218 Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
219 Eq(std::numeric_limits<int>::max())),
220 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
221 Eq(absl::nullopt)));
222}
223
224auto FpsMax() {
225 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
226 Eq(kDefaultFramerate));
227}
228
229auto FpsUnlimited() {
230 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
231 Eq(std::numeric_limits<int>::max()));
232}
233
234auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
235 return AllOf(Field("max_framerate_fps",
236 &rtc::VideoSinkWants::max_framerate_fps, fps_matcher),
237 ResolutionMax());
238}
239
240auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
241 return AllOf(FpsMax(),
242 Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
243 AllOf(pixel_matcher, Gt(0))));
244}
245
246auto FpsMaxResolutionMax() {
247 return AllOf(FpsMax(), ResolutionMax());
248}
249
250auto UnlimitedSinkWants() {
251 return AllOf(FpsUnlimited(), ResolutionMax());
252}
253
254auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
255 Matcher<int> fps_range_matcher;
256
257 if (last_frame_pixels <= 320 * 240) {
258 fps_range_matcher = AllOf(Ge(7), Le(10));
259 } else if (last_frame_pixels <= 480 * 270) {
260 fps_range_matcher = AllOf(Ge(10), Le(15));
261 } else if (last_frame_pixels <= 640 * 480) {
262 fps_range_matcher = Ge(15);
263 } else {
264 fps_range_matcher = Eq(kDefaultFramerate);
265 }
266 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
267 fps_range_matcher);
268}
269
mflodmancc3d4422017-08-03 08:27:51 -0700270class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700271 public:
Niels Möller213618e2018-07-24 09:29:58 +0200272 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200273 const VideoStreamEncoderSettings& settings,
274 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100275 : VideoStreamEncoder(Clock::GetRealTimeClock(),
276 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200277 stats_proxy,
278 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200279 std::unique_ptr<OveruseFrameDetector>(
280 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100281 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100282 task_queue_factory),
Henrik Boströmc55516d2020-05-11 16:29:22 +0200283 fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
284 fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
Henrik Boström381d1092020-05-12 18:49:07 +0200285 fake_cpu_resource_->Initialize(encoder_queue(),
286 resource_adaptation_queue());
287 fake_quality_resource_->Initialize(encoder_queue(),
288 resource_adaptation_queue());
Henrik Boströmc55516d2020-05-11 16:29:22 +0200289 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200290 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200291 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100292 }
perkj803d97f2016-11-01 11:45:46 -0700293
Henrik Boström381d1092020-05-12 18:49:07 +0200294 void SetSourceAndWaitForRestrictionsUpdated(
295 rtc::VideoSourceInterface<VideoFrame>* source,
296 const DegradationPreference& degradation_preference) {
297 VideoSourceRestrictionsUpdatedListener listener;
298 AddAdaptationListenerForTesting(&listener);
299 SetSource(source, degradation_preference);
300 listener.restrictions_updated_event()->Wait(5000);
301 RemoveAdaptationListenerForTesting(&listener);
302 }
303
304 void SetSourceAndWaitForFramerateUpdated(
305 rtc::VideoSourceInterface<VideoFrame>* source,
306 const DegradationPreference& degradation_preference) {
307 overuse_detector_proxy_->framerate_updated_event()->Reset();
308 SetSource(source, degradation_preference);
309 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
310 }
311
312 void OnBitrateUpdatedAndWaitForManagedResources(
313 DataRate target_bitrate,
314 DataRate stable_target_bitrate,
315 DataRate link_allocation,
316 uint8_t fraction_lost,
317 int64_t round_trip_time_ms,
318 double cwnd_reduce_ratio) {
319 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
320 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
321 // Bitrate is updated on the encoder queue.
322 WaitUntilTaskQueueIsIdle();
323 // Give the managed resources time to react to the new bitrate.
324 // TODO(hbos): Can we await an appropriate event instead?
325 WaitUntilAdaptationTaskQueueIsIdle();
326 }
327
328 void WaitUntilAdaptationTaskQueueIsIdle() {
329 rtc::Event event;
330 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
331 ASSERT_TRUE(event.Wait(5000));
332 }
333
kthelgason2fc52542017-03-03 00:24:41 -0800334 // This is used as a synchronisation mechanism, to make sure that the
335 // encoder queue is not blocked before we start sending it frames.
336 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100337 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200338 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800339 ASSERT_TRUE(event.Wait(5000));
340 }
341
Henrik Boström91aa7322020-04-28 12:24:33 +0200342 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200343 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200344 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200345 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200346 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
347 event.Set();
348 });
349 ASSERT_TRUE(event.Wait(5000));
350 }
351 void TriggerCpuUnderuse() {
352 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200353 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200354 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
355 event.Set();
356 });
357 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200358 }
kthelgason876222f2016-11-29 01:44:11 -0800359
Henrik Boström91aa7322020-04-28 12:24:33 +0200360 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200361 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200362 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200363 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200364 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
365 event.Set();
366 });
367 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200368 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200369 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200370 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200371 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200372 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
373 event.Set();
374 });
375 ASSERT_TRUE(event.Wait(5000));
376 }
377
378 // Fakes high QP resource usage measurements on the real
379 // QualityScalerResource. Returns whether or not QP samples would have been
380 // cleared if this had been a real signal from the QualityScaler.
381 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
383 new FakeQualityScalerQpUsageHandlerCallback();
Henrik Boström5bf60e42020-05-13 15:20:25 +0200384 encoder_queue()->PostTask([this, callback] {
385 // This will cause a "ping" between adaptation task queue and encoder
386 // queue. When we have the result, the |callback| will be notified.
Henrik Boström91aa7322020-04-28 12:24:33 +0200387 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
Henrik Boström91aa7322020-04-28 12:24:33 +0200388 });
Henrik Boström5bf60e42020-05-13 15:20:25 +0200389 EXPECT_TRUE(callback->WaitForQpUsageHandled());
Henrik Boström91aa7322020-04-28 12:24:33 +0200390 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
391 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200392 }
sprangfda496a2017-06-15 04:21:07 -0700393
Niels Möller7dc26b72017-12-06 10:27:48 +0100394 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200395 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
396 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700397};
398
asapersson5f7226f2016-11-25 04:37:00 -0800399class VideoStreamFactory
400 : public VideoEncoderConfig::VideoStreamFactoryInterface {
401 public:
sprangfda496a2017-06-15 04:21:07 -0700402 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
403 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800404 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700405 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800406 }
407
408 private:
409 std::vector<VideoStream> CreateEncoderStreams(
410 int width,
411 int height,
412 const VideoEncoderConfig& encoder_config) override {
413 std::vector<VideoStream> streams =
414 test::CreateVideoStreams(width, height, encoder_config);
415 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100416 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700417 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800418 }
419 return streams;
420 }
sprangfda496a2017-06-15 04:21:07 -0700421
asapersson5f7226f2016-11-25 04:37:00 -0800422 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700423 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800424};
425
Noah Richards51db4212019-06-12 06:59:12 -0700426// Simulates simulcast behavior and makes highest stream resolutions divisible
427// by 4.
428class CroppingVideoStreamFactory
429 : public VideoEncoderConfig::VideoStreamFactoryInterface {
430 public:
431 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
432 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
433 EXPECT_GT(num_temporal_layers, 0u);
434 EXPECT_GT(framerate, 0);
435 }
436
437 private:
438 std::vector<VideoStream> CreateEncoderStreams(
439 int width,
440 int height,
441 const VideoEncoderConfig& encoder_config) override {
442 std::vector<VideoStream> streams = test::CreateVideoStreams(
443 width - width % 4, height - height % 4, encoder_config);
444 for (VideoStream& stream : streams) {
445 stream.num_temporal_layers = num_temporal_layers_;
446 stream.max_framerate = framerate_;
447 }
448 return streams;
449 }
450
451 const size_t num_temporal_layers_;
452 const int framerate_;
453};
454
sprangb1ca0732017-02-01 08:38:12 -0800455class AdaptingFrameForwarder : public test::FrameForwarder {
456 public:
457 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700458 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800459
460 void set_adaptation_enabled(bool enabled) {
461 rtc::CritScope cs(&crit_);
462 adaptation_enabled_ = enabled;
463 }
464
asaperssonfab67072017-04-04 05:51:49 -0700465 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800466 rtc::CritScope cs(&crit_);
467 return adaptation_enabled_;
468 }
469
asapersson09f05612017-05-15 23:40:18 -0700470 rtc::VideoSinkWants last_wants() const {
471 rtc::CritScope cs(&crit_);
472 return last_wants_;
473 }
474
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200475 absl::optional<int> last_sent_width() const { return last_width_; }
476 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800477
sprangb1ca0732017-02-01 08:38:12 -0800478 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
479 int cropped_width = 0;
480 int cropped_height = 0;
481 int out_width = 0;
482 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700483 if (adaption_enabled()) {
484 if (adapter_.AdaptFrameResolution(
485 video_frame.width(), video_frame.height(),
486 video_frame.timestamp_us() * 1000, &cropped_width,
487 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100488 VideoFrame adapted_frame =
489 VideoFrame::Builder()
490 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
491 nullptr, out_width, out_height))
492 .set_timestamp_rtp(99)
493 .set_timestamp_ms(99)
494 .set_rotation(kVideoRotation_0)
495 .build();
sprangc5d62e22017-04-02 23:53:04 -0700496 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100497 if (video_frame.has_update_rect()) {
498 adapted_frame.set_update_rect(
499 video_frame.update_rect().ScaleWithFrame(
500 video_frame.width(), video_frame.height(), 0, 0,
501 video_frame.width(), video_frame.height(), out_width,
502 out_height));
503 }
sprangc5d62e22017-04-02 23:53:04 -0700504 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800505 last_width_.emplace(adapted_frame.width());
506 last_height_.emplace(adapted_frame.height());
507 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200508 last_width_ = absl::nullopt;
509 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700510 }
sprangb1ca0732017-02-01 08:38:12 -0800511 } else {
512 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800513 last_width_.emplace(video_frame.width());
514 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800515 }
516 }
517
518 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
519 const rtc::VideoSinkWants& wants) override {
520 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700521 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100522 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800523 test::FrameForwarder::AddOrUpdateSink(sink, wants);
524 }
sprangb1ca0732017-02-01 08:38:12 -0800525 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700526 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
527 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200528 absl::optional<int> last_width_;
529 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800530};
sprangc5d62e22017-04-02 23:53:04 -0700531
Niels Möller213618e2018-07-24 09:29:58 +0200532// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700533class MockableSendStatisticsProxy : public SendStatisticsProxy {
534 public:
535 MockableSendStatisticsProxy(Clock* clock,
536 const VideoSendStream::Config& config,
537 VideoEncoderConfig::ContentType content_type)
538 : SendStatisticsProxy(clock, config, content_type) {}
539
540 VideoSendStream::Stats GetStats() override {
541 rtc::CritScope cs(&lock_);
542 if (mock_stats_)
543 return *mock_stats_;
544 return SendStatisticsProxy::GetStats();
545 }
546
Niels Möller213618e2018-07-24 09:29:58 +0200547 int GetInputFrameRate() const override {
548 rtc::CritScope cs(&lock_);
549 if (mock_stats_)
550 return mock_stats_->input_frame_rate;
551 return SendStatisticsProxy::GetInputFrameRate();
552 }
sprangc5d62e22017-04-02 23:53:04 -0700553 void SetMockStats(const VideoSendStream::Stats& stats) {
554 rtc::CritScope cs(&lock_);
555 mock_stats_.emplace(stats);
556 }
557
558 void ResetMockStats() {
559 rtc::CritScope cs(&lock_);
560 mock_stats_.reset();
561 }
562
563 private:
564 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200565 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700566};
567
sprang4847ae62017-06-27 07:06:52 -0700568class MockBitrateObserver : public VideoBitrateAllocationObserver {
569 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200570 MOCK_METHOD(void,
571 OnBitrateAllocationUpdated,
572 (const VideoBitrateAllocation&),
573 (override));
sprang4847ae62017-06-27 07:06:52 -0700574};
575
philipel9b058032020-02-10 11:30:00 +0100576class MockEncoderSelector
577 : public VideoEncoderFactory::EncoderSelectorInterface {
578 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200579 MOCK_METHOD(void,
580 OnCurrentEncoder,
581 (const SdpVideoFormat& format),
582 (override));
583 MOCK_METHOD(absl::optional<SdpVideoFormat>,
584 OnAvailableBitrate,
585 (const DataRate& rate),
586 (override));
587 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100588};
589
perkj803d97f2016-11-01 11:45:46 -0700590} // namespace
591
mflodmancc3d4422017-08-03 08:27:51 -0700592class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700593 public:
594 static const int kDefaultTimeoutMs = 30 * 1000;
595
mflodmancc3d4422017-08-03 08:27:51 -0700596 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700597 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700598 codec_width_(320),
599 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200600 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200601 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700602 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200603 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700604 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700605 Clock::GetRealTimeClock(),
606 video_send_config_,
607 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700608 sink_(&fake_encoder_) {}
609
610 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700611 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700612 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200613 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800614 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200615 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200616 video_send_config_.rtp.payload_name = "FAKE";
617 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700618
Per512ecb32016-09-23 15:52:06 +0200619 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200620 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700621 video_encoder_config.video_stream_factory =
622 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100623 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700624
625 // Framerate limit is specified by the VideoStreamFactory.
626 std::vector<VideoStream> streams =
627 video_encoder_config.video_stream_factory->CreateEncoderStreams(
628 codec_width_, codec_height_, video_encoder_config);
629 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100630 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700631
Niels Möllerf1338562018-04-26 09:51:47 +0200632 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800633 }
634
Niels Möllerf1338562018-04-26 09:51:47 +0200635 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700636 if (video_stream_encoder_)
637 video_stream_encoder_->Stop();
638 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200639 stats_proxy_.get(), video_send_config_.encoder_settings,
640 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700641 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
642 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700643 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700644 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
645 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200646 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700647 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800648 }
649
650 void ResetEncoder(const std::string& payload_name,
651 size_t num_streams,
652 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700653 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700654 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200655 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800656
657 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200658 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800659 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100660 video_encoder_config.max_bitrate_bps =
661 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800662 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700663 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
664 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700665 video_encoder_config.content_type =
666 screenshare ? VideoEncoderConfig::ContentType::kScreen
667 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700668 if (payload_name == "VP9") {
669 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
670 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
671 video_encoder_config.encoder_specific_settings =
672 new rtc::RefCountedObject<
673 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
674 }
Niels Möllerf1338562018-04-26 09:51:47 +0200675 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700676 }
677
sprang57c2fff2017-01-16 06:24:02 -0800678 VideoFrame CreateFrame(int64_t ntp_time_ms,
679 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100680 VideoFrame frame =
681 VideoFrame::Builder()
682 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
683 destruction_event, codec_width_, codec_height_))
684 .set_timestamp_rtp(99)
685 .set_timestamp_ms(99)
686 .set_rotation(kVideoRotation_0)
687 .build();
sprang57c2fff2017-01-16 06:24:02 -0800688 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700689 return frame;
690 }
691
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100692 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
693 rtc::Event* destruction_event,
694 int offset_x) const {
695 VideoFrame frame =
696 VideoFrame::Builder()
697 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
698 destruction_event, codec_width_, codec_height_))
699 .set_timestamp_rtp(99)
700 .set_timestamp_ms(99)
701 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100702 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100703 .build();
704 frame.set_ntp_time_ms(ntp_time_ms);
705 return frame;
706 }
707
sprang57c2fff2017-01-16 06:24:02 -0800708 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100709 VideoFrame frame =
710 VideoFrame::Builder()
711 .set_video_frame_buffer(
712 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
713 .set_timestamp_rtp(99)
714 .set_timestamp_ms(99)
715 .set_rotation(kVideoRotation_0)
716 .build();
sprang57c2fff2017-01-16 06:24:02 -0800717 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700718 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700719 return frame;
720 }
721
Noah Richards51db4212019-06-12 06:59:12 -0700722 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
723 rtc::Event* destruction_event,
724 int width,
725 int height) const {
726 VideoFrame frame =
727 VideoFrame::Builder()
728 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
729 destruction_event, width, height))
730 .set_timestamp_rtp(99)
731 .set_timestamp_ms(99)
732 .set_rotation(kVideoRotation_0)
733 .build();
734 frame.set_ntp_time_ms(ntp_time_ms);
735 return frame;
736 }
737
738 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
739 rtc::Event* destruction_event) const {
740 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
741 codec_height_);
742 }
743
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100744 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
745 MockBitrateObserver bitrate_observer;
746 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
747
748 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
749 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200750 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100751 DataRate::BitsPerSec(kTargetBitrateBps),
752 DataRate::BitsPerSec(kTargetBitrateBps),
753 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100754
755 video_source_.IncomingCapturedFrame(
756 CreateFrame(1, codec_width_, codec_height_));
757 WaitForEncodedFrame(1);
758 }
759
asapersson09f05612017-05-15 23:40:18 -0700760 void VerifyFpsEqResolutionEq(const rtc::VideoSinkWants& wants1,
761 const rtc::VideoSinkWants& wants2) {
762 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
763 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
764 }
765
766 void VerifyFpsMaxResolutionLt(const rtc::VideoSinkWants& wants1,
767 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200768 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700769 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
770 EXPECT_GT(wants1.max_pixel_count, 0);
771 }
772
773 void VerifyFpsMaxResolutionGt(const rtc::VideoSinkWants& wants1,
774 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200775 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asapersson09f05612017-05-15 23:40:18 -0700776 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
777 }
778
asaperssonf7e294d2017-06-13 23:25:22 -0700779 void VerifyFpsMaxResolutionEq(const rtc::VideoSinkWants& wants1,
780 const rtc::VideoSinkWants& wants2) {
Åsa Persson8c1bf952018-09-13 10:42:19 +0200781 EXPECT_EQ(kDefaultFramerate, wants1.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -0700782 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
783 }
784
785 void VerifyFpsLtResolutionEq(const rtc::VideoSinkWants& wants1,
786 const rtc::VideoSinkWants& wants2) {
787 EXPECT_LT(wants1.max_framerate_fps, wants2.max_framerate_fps);
788 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
789 }
790
791 void VerifyFpsGtResolutionEq(const rtc::VideoSinkWants& wants1,
792 const rtc::VideoSinkWants& wants2) {
793 EXPECT_GT(wants1.max_framerate_fps, wants2.max_framerate_fps);
794 EXPECT_EQ(wants1.max_pixel_count, wants2.max_pixel_count);
795 }
796
797 void VerifyFpsEqResolutionLt(const rtc::VideoSinkWants& wants1,
798 const rtc::VideoSinkWants& wants2) {
799 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
800 EXPECT_LT(wants1.max_pixel_count, wants2.max_pixel_count);
801 EXPECT_GT(wants1.max_pixel_count, 0);
802 }
803
804 void VerifyFpsEqResolutionGt(const rtc::VideoSinkWants& wants1,
805 const rtc::VideoSinkWants& wants2) {
806 EXPECT_EQ(wants1.max_framerate_fps, wants2.max_framerate_fps);
807 EXPECT_GT(wants1.max_pixel_count, wants2.max_pixel_count);
808 }
809
sprang4847ae62017-06-27 07:06:52 -0700810 void WaitForEncodedFrame(int64_t expected_ntp_time) {
811 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100812 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700813 }
814
815 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
816 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100817 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700818 return ok;
819 }
820
821 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
822 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100823 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700824 }
825
826 void ExpectDroppedFrame() {
827 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100828 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700829 }
830
831 bool WaitForFrame(int64_t timeout_ms) {
832 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100833 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700834 return ok;
835 }
836
perkj26091b12016-09-01 01:17:40 -0700837 class TestEncoder : public test::FakeEncoder {
838 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100839 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700840
asaperssonfab67072017-04-04 05:51:49 -0700841 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800842 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700843 return config_;
844 }
845
846 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800847 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700848 block_next_encode_ = true;
849 }
850
Erik Språngaed30702018-11-05 12:57:17 +0100851 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800852 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100853 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100854 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100855 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100856 info.scaling_settings = VideoEncoder::ScalingSettings(
857 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100858 }
859 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100860 for (int i = 0; i < kMaxSpatialLayers; ++i) {
861 if (temporal_layers_supported_[i]) {
862 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
863 info.fps_allocation[i].resize(num_layers);
864 }
865 }
Erik Språngaed30702018-11-05 12:57:17 +0100866 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200867
868 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100869 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100870 return info;
kthelgason876222f2016-11-29 01:44:11 -0800871 }
872
Erik Språngb7cb7b52019-02-26 15:52:33 +0100873 int32_t RegisterEncodeCompleteCallback(
874 EncodedImageCallback* callback) override {
875 rtc::CritScope lock(&local_crit_sect_);
876 encoded_image_callback_ = callback;
877 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
878 }
879
perkjfa10b552016-10-02 23:45:26 -0700880 void ContinueEncode() { continue_encode_event_.Set(); }
881
882 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
883 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800884 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700885 EXPECT_EQ(timestamp_, timestamp);
886 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
887 }
888
kthelgason2fc52542017-03-03 00:24:41 -0800889 void SetQualityScaling(bool b) {
890 rtc::CritScope lock(&local_crit_sect_);
891 quality_scaling_ = b;
892 }
kthelgasonad9010c2017-02-14 00:46:51 -0800893
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100894 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
895 rtc::CritScope lock(&local_crit_sect_);
896 requested_resolution_alignment_ = requested_resolution_alignment;
897 }
898
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100899 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
900 rtc::CritScope lock(&local_crit_sect_);
901 is_hardware_accelerated_ = is_hardware_accelerated;
902 }
903
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100904 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
905 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
906 rtc::CritScope lock(&local_crit_sect_);
907 temporal_layers_supported_[spatial_idx] = supported;
908 }
909
Sergey Silkin6456e352019-07-08 17:56:40 +0200910 void SetResolutionBitrateLimits(
911 std::vector<ResolutionBitrateLimits> thresholds) {
912 rtc::CritScope cs(&local_crit_sect_);
913 resolution_bitrate_limits_ = thresholds;
914 }
915
sprangfe627f32017-03-29 08:24:59 -0700916 void ForceInitEncodeFailure(bool force_failure) {
917 rtc::CritScope lock(&local_crit_sect_);
918 force_init_encode_failed_ = force_failure;
919 }
920
Niels Möller6bb5ab92019-01-11 11:11:10 +0100921 void SimulateOvershoot(double rate_factor) {
922 rtc::CritScope lock(&local_crit_sect_);
923 rate_factor_ = rate_factor;
924 }
925
Erik Språngd7329ca2019-02-21 21:19:53 +0100926 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100927 rtc::CritScope lock(&local_crit_sect_);
928 return last_framerate_;
929 }
930
Erik Språngd7329ca2019-02-21 21:19:53 +0100931 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100932 rtc::CritScope lock(&local_crit_sect_);
933 return last_update_rect_;
934 }
935
Niels Möller87e2d782019-03-07 10:18:23 +0100936 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100937 rtc::CritScope lock(&local_crit_sect_);
938 return last_frame_types_;
939 }
940
941 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100942 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100943 keyframe ? VideoFrameType::kVideoFrameKey
944 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100945 {
946 rtc::CritScope lock(&local_crit_sect_);
947 last_frame_types_ = frame_type;
948 }
Niels Möllerb859b322019-03-07 12:40:01 +0100949 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100950 }
951
Erik Språngb7cb7b52019-02-26 15:52:33 +0100952 void InjectEncodedImage(const EncodedImage& image) {
953 rtc::CritScope lock(&local_crit_sect_);
954 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
955 }
956
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200957 void InjectEncodedImage(const EncodedImage& image,
958 const CodecSpecificInfo* codec_specific_info,
959 const RTPFragmentationHeader* fragmentation) {
960 rtc::CritScope lock(&local_crit_sect_);
961 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
962 fragmentation);
963 }
964
Erik Språngd7329ca2019-02-21 21:19:53 +0100965 void ExpectNullFrame() {
966 rtc::CritScope lock(&local_crit_sect_);
967 expect_null_frame_ = true;
968 }
969
Erik Språng5056af02019-09-02 15:53:11 +0200970 absl::optional<VideoEncoder::RateControlParameters>
971 GetAndResetLastRateControlSettings() {
972 auto settings = last_rate_control_settings_;
973 last_rate_control_settings_.reset();
974 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100975 }
976
Sergey Silkin5ee69672019-07-02 14:18:34 +0200977 int GetNumEncoderInitializations() const {
978 rtc::CritScope lock(&local_crit_sect_);
979 return num_encoder_initializations_;
980 }
981
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200982 int GetNumSetRates() const {
983 rtc::CritScope lock(&local_crit_sect_);
984 return num_set_rates_;
985 }
986
perkjfa10b552016-10-02 23:45:26 -0700987 private:
perkj26091b12016-09-01 01:17:40 -0700988 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100989 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700990 bool block_encode;
991 {
brandtre78d2662017-01-16 05:57:16 -0800992 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100993 if (expect_null_frame_) {
994 EXPECT_EQ(input_image.timestamp(), 0u);
995 EXPECT_EQ(input_image.width(), 1);
996 last_frame_types_ = *frame_types;
997 expect_null_frame_ = false;
998 } else {
999 EXPECT_GT(input_image.timestamp(), timestamp_);
1000 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1001 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1002 }
perkj26091b12016-09-01 01:17:40 -07001003
1004 timestamp_ = input_image.timestamp();
1005 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001006 last_input_width_ = input_image.width();
1007 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001008 block_encode = block_next_encode_;
1009 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001010 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001011 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -07001012 }
Niels Möllerb859b322019-03-07 12:40:01 +01001013 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001014 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001015 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001016 return result;
1017 }
1018
sprangfe627f32017-03-29 08:24:59 -07001019 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001020 const Settings& settings) override {
1021 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001022
sprangfe627f32017-03-29 08:24:59 -07001023 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001024 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001025
1026 ++num_encoder_initializations_;
1027
Erik Språng82fad3d2018-03-21 09:57:23 +01001028 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001029 // Simulate setting up temporal layers, in order to validate the life
1030 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001031 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001032 frame_buffer_controller_ =
1033 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001034 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001035 if (force_init_encode_failed_) {
1036 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001037 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001038 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001039
Erik Språngb7cb7b52019-02-26 15:52:33 +01001040 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001041 return res;
1042 }
1043
Erik Språngb7cb7b52019-02-26 15:52:33 +01001044 int32_t Release() override {
1045 rtc::CritScope lock(&local_crit_sect_);
1046 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1047 initialized_ = EncoderState::kUninitialized;
1048 return FakeEncoder::Release();
1049 }
1050
Erik Språng16cb8f52019-04-12 13:59:09 +02001051 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001052 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001053 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001054 VideoBitrateAllocation adjusted_rate_allocation;
1055 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1056 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001057 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001058 adjusted_rate_allocation.SetBitrate(
1059 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001060 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001061 rate_factor_));
1062 }
1063 }
1064 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001065 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001066 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001067 RateControlParameters adjusted_paramters = parameters;
1068 adjusted_paramters.bitrate = adjusted_rate_allocation;
1069 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001070 }
1071
brandtre78d2662017-01-16 05:57:16 -08001072 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001073 enum class EncoderState {
1074 kUninitialized,
1075 kInitializationFailed,
1076 kInitialized
1077 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
1078 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -07001079 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -07001080 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -07001081 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1082 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1083 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1084 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1085 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001086 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001087 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +01001088 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -07001089 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001090 absl::optional<bool>
1091 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
1092 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -07001093 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001094 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
1095 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001096 absl::optional<VideoEncoder::RateControlParameters>
1097 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001098 VideoFrame::UpdateRect last_update_rect_
1099 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001100 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001101 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001102 EncodedImageCallback* encoded_image_callback_
1103 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001104 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001105 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001106 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1107 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001108 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001109 };
1110
mflodmancc3d4422017-08-03 08:27:51 -07001111 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001112 public:
1113 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001114 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001115
perkj26091b12016-09-01 01:17:40 -07001116 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001117 EXPECT_TRUE(
1118 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1119 }
1120
1121 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1122 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001123 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001124 if (!encoded_frame_event_.Wait(timeout_ms))
1125 return false;
perkj26091b12016-09-01 01:17:40 -07001126 {
1127 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001128 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001129 }
1130 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001131 return true;
perkj26091b12016-09-01 01:17:40 -07001132 }
1133
sprangb1ca0732017-02-01 08:38:12 -08001134 void WaitForEncodedFrame(uint32_t expected_width,
1135 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001136 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001137 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001138 }
1139
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001140 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001141 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001142 uint32_t width = 0;
1143 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001144 {
1145 rtc::CritScope lock(&crit_);
1146 width = last_width_;
1147 height = last_height_;
1148 }
1149 EXPECT_EQ(expected_height, height);
1150 EXPECT_EQ(expected_width, width);
1151 }
1152
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001153 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1154 int width = 0;
1155 int height = 0;
1156 {
1157 rtc::CritScope lock(&crit_);
1158 width = last_width_;
1159 height = last_height_;
1160 }
1161 EXPECT_EQ(width % resolution_alignment, 0);
1162 EXPECT_EQ(height % resolution_alignment, 0);
1163 }
1164
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001165 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1166 VideoRotation rotation;
1167 {
1168 rtc::CritScope lock(&crit_);
1169 rotation = last_rotation_;
1170 }
1171 EXPECT_EQ(expected_rotation, rotation);
1172 }
1173
kthelgason2fc52542017-03-03 00:24:41 -08001174 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001175
sprangc5d62e22017-04-02 23:53:04 -07001176 bool WaitForFrame(int64_t timeout_ms) {
1177 return encoded_frame_event_.Wait(timeout_ms);
1178 }
1179
perkj26091b12016-09-01 01:17:40 -07001180 void SetExpectNoFrames() {
1181 rtc::CritScope lock(&crit_);
1182 expect_frames_ = false;
1183 }
1184
asaperssonfab67072017-04-04 05:51:49 -07001185 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001186 rtc::CritScope lock(&crit_);
1187 return number_of_reconfigurations_;
1188 }
1189
asaperssonfab67072017-04-04 05:51:49 -07001190 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001191 rtc::CritScope lock(&crit_);
1192 return min_transmit_bitrate_bps_;
1193 }
1194
Erik Språngd7329ca2019-02-21 21:19:53 +01001195 void SetNumExpectedLayers(size_t num_layers) {
1196 rtc::CritScope lock(&crit_);
1197 num_expected_layers_ = num_layers;
1198 }
1199
Erik Språngb7cb7b52019-02-26 15:52:33 +01001200 int64_t GetLastCaptureTimeMs() const {
1201 rtc::CritScope lock(&crit_);
1202 return last_capture_time_ms_;
1203 }
1204
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001205 std::vector<uint8_t> GetLastEncodedImageData() {
1206 rtc::CritScope lock(&crit_);
1207 return std::move(last_encoded_image_data_);
1208 }
1209
1210 RTPFragmentationHeader GetLastFragmentation() {
1211 rtc::CritScope lock(&crit_);
1212 return std::move(last_fragmentation_);
1213 }
1214
perkj26091b12016-09-01 01:17:40 -07001215 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001216 Result OnEncodedImage(
1217 const EncodedImage& encoded_image,
1218 const CodecSpecificInfo* codec_specific_info,
1219 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001220 rtc::CritScope lock(&crit_);
1221 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001222 last_encoded_image_data_ = std::vector<uint8_t>(
1223 encoded_image.data(), encoded_image.data() + encoded_image.size());
1224 if (fragmentation) {
1225 last_fragmentation_.CopyFrom(*fragmentation);
1226 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001227 uint32_t timestamp = encoded_image.Timestamp();
1228 if (last_timestamp_ != timestamp) {
1229 num_received_layers_ = 1;
1230 } else {
1231 ++num_received_layers_;
1232 }
1233 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001234 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001235 last_width_ = encoded_image._encodedWidth;
1236 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001237 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001238 if (num_received_layers_ == num_expected_layers_) {
1239 encoded_frame_event_.Set();
1240 }
sprangb1ca0732017-02-01 08:38:12 -08001241 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001242 }
1243
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001244 void OnEncoderConfigurationChanged(
1245 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001246 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001247 VideoEncoderConfig::ContentType content_type,
1248 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001249 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001250 ++number_of_reconfigurations_;
1251 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1252 }
1253
perkj26091b12016-09-01 01:17:40 -07001254 rtc::CriticalSection crit_;
1255 TestEncoder* test_encoder_;
1256 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001257 std::vector<uint8_t> last_encoded_image_data_;
1258 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001259 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001260 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001261 uint32_t last_height_ = 0;
1262 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001263 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001264 size_t num_expected_layers_ = 1;
1265 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001266 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001267 int number_of_reconfigurations_ = 0;
1268 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001269 };
1270
Sergey Silkin5ee69672019-07-02 14:18:34 +02001271 class VideoBitrateAllocatorProxyFactory
1272 : public VideoBitrateAllocatorFactory {
1273 public:
1274 VideoBitrateAllocatorProxyFactory()
1275 : bitrate_allocator_factory_(
1276 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1277
1278 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1279 const VideoCodec& codec) override {
1280 rtc::CritScope lock(&crit_);
1281 codec_config_ = codec;
1282 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1283 }
1284
1285 VideoCodec codec_config() const {
1286 rtc::CritScope lock(&crit_);
1287 return codec_config_;
1288 }
1289
1290 private:
1291 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1292
1293 rtc::CriticalSection crit_;
1294 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1295 };
1296
perkj26091b12016-09-01 01:17:40 -07001297 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001298 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001299 int codec_width_;
1300 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001301 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001302 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001303 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001304 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001305 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001306 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001307 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001308 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001309 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001310 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001311};
1312
mflodmancc3d4422017-08-03 08:27:51 -07001313TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001314 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001315 DataRate::BitsPerSec(kTargetBitrateBps),
1316 DataRate::BitsPerSec(kTargetBitrateBps),
1317 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001318 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001319 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001320 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001321 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001322 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001323}
1324
mflodmancc3d4422017-08-03 08:27:51 -07001325TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001326 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001327 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001328 // The encoder will cache up to one frame for a short duration. Adding two
1329 // frames means that the first frame will be dropped and the second frame will
1330 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001331 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001332 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001333 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001334
Henrik Boström381d1092020-05-12 18:49:07 +02001335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001336 DataRate::BitsPerSec(kTargetBitrateBps),
1337 DataRate::BitsPerSec(kTargetBitrateBps),
1338 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001339
Sebastian Janssona3177052018-04-10 13:05:49 +02001340 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001341 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001342 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1343
1344 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001345 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001346}
1347
mflodmancc3d4422017-08-03 08:27:51 -07001348TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001349 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001350 DataRate::BitsPerSec(kTargetBitrateBps),
1351 DataRate::BitsPerSec(kTargetBitrateBps),
1352 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001353 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001354 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001355
Henrik Boström381d1092020-05-12 18:49:07 +02001356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1357 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1358 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001359 // The encoder will cache up to one frame for a short duration. Adding two
1360 // frames means that the first frame will be dropped and the second frame will
1361 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001362 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001363 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001364
Henrik Boström381d1092020-05-12 18:49:07 +02001365 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001366 DataRate::BitsPerSec(kTargetBitrateBps),
1367 DataRate::BitsPerSec(kTargetBitrateBps),
1368 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001369 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001370 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1371 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001372 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001373}
1374
mflodmancc3d4422017-08-03 08:27:51 -07001375TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001377 DataRate::BitsPerSec(kTargetBitrateBps),
1378 DataRate::BitsPerSec(kTargetBitrateBps),
1379 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001380 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001381 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001382
1383 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001384 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001385
perkja49cbd32016-09-16 07:53:41 -07001386 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001387 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001388 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001389}
1390
mflodmancc3d4422017-08-03 08:27:51 -07001391TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001393 DataRate::BitsPerSec(kTargetBitrateBps),
1394 DataRate::BitsPerSec(kTargetBitrateBps),
1395 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001396
perkja49cbd32016-09-16 07:53:41 -07001397 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001398 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001399
mflodmancc3d4422017-08-03 08:27:51 -07001400 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001401 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001402 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001403 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1404 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001405}
1406
mflodmancc3d4422017-08-03 08:27:51 -07001407TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001408 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001409 DataRate::BitsPerSec(kTargetBitrateBps),
1410 DataRate::BitsPerSec(kTargetBitrateBps),
1411 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001412
1413 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001414 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001415 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001416 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1417 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001418 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1419 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001420 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001421 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001422
mflodmancc3d4422017-08-03 08:27:51 -07001423 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001424}
1425
Noah Richards51db4212019-06-12 06:59:12 -07001426TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001428 DataRate::BitsPerSec(kTargetBitrateBps),
1429 DataRate::BitsPerSec(kTargetBitrateBps),
1430 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001431
1432 rtc::Event frame_destroyed_event;
1433 video_source_.IncomingCapturedFrame(
1434 CreateFakeNativeFrame(1, &frame_destroyed_event));
1435 ExpectDroppedFrame();
1436 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1437 video_stream_encoder_->Stop();
1438}
1439
1440TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1441 // Use the cropping factory.
1442 video_encoder_config_.video_stream_factory =
1443 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1444 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1445 kMaxPayloadLength);
1446 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1447
1448 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001450 DataRate::BitsPerSec(kTargetBitrateBps),
1451 DataRate::BitsPerSec(kTargetBitrateBps),
1452 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001453 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1454 WaitForEncodedFrame(1);
1455 // The encoder will have been configured once.
1456 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1457 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1458 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1459
1460 // Now send in a fake frame that needs to be cropped as the width/height
1461 // aren't divisible by 4 (see CreateEncoderStreams above).
1462 rtc::Event frame_destroyed_event;
1463 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1464 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1465 ExpectDroppedFrame();
1466 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1467 video_stream_encoder_->Stop();
1468}
1469
Ying Wang9b881ab2020-02-07 14:29:32 +01001470TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001472 DataRate::BitsPerSec(kTargetBitrateBps),
1473 DataRate::BitsPerSec(kTargetBitrateBps),
1474 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001475 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1476 WaitForEncodedFrame(1);
1477
Henrik Boström381d1092020-05-12 18:49:07 +02001478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001479 DataRate::BitsPerSec(kTargetBitrateBps),
1480 DataRate::BitsPerSec(kTargetBitrateBps),
1481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001482 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1483 // frames. Adding two frames means that the first frame will be dropped and
1484 // the second frame will be sent to the encoder.
1485 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1486 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1487 WaitForEncodedFrame(3);
1488 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1489 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1490 WaitForEncodedFrame(5);
1491 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1492 video_stream_encoder_->Stop();
1493}
1494
mflodmancc3d4422017-08-03 08:27:51 -07001495TEST_F(VideoStreamEncoderTest,
1496 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001498 DataRate::BitsPerSec(kTargetBitrateBps),
1499 DataRate::BitsPerSec(kTargetBitrateBps),
1500 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001501 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001502
1503 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001504 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001505 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001506 // The encoder will have been configured once when the first frame is
1507 // received.
1508 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001509
1510 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001511 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001512 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001513 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001514 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001515
1516 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001517 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001518 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001519 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001520 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001521
mflodmancc3d4422017-08-03 08:27:51 -07001522 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001523}
1524
mflodmancc3d4422017-08-03 08:27:51 -07001525TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001527 DataRate::BitsPerSec(kTargetBitrateBps),
1528 DataRate::BitsPerSec(kTargetBitrateBps),
1529 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001530
1531 // Capture a frame and wait for it to synchronize with the encoder thread.
1532 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001533 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001534 // The encoder will have been configured once.
1535 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001536 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1537 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1538
1539 codec_width_ *= 2;
1540 codec_height_ *= 2;
1541 // Capture a frame with a higher resolution and wait for it to synchronize
1542 // with the encoder thread.
1543 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001544 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001545 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1546 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001547 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001548
mflodmancc3d4422017-08-03 08:27:51 -07001549 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001550}
1551
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001552TEST_F(VideoStreamEncoderTest,
1553 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001555 DataRate::BitsPerSec(kTargetBitrateBps),
1556 DataRate::BitsPerSec(kTargetBitrateBps),
1557 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001558
1559 // Capture a frame and wait for it to synchronize with the encoder thread.
1560 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1561 WaitForEncodedFrame(1);
1562
1563 VideoEncoderConfig video_encoder_config;
1564 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1565 // Changing the max payload data length recreates encoder.
1566 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1567 kMaxPayloadLength / 2);
1568
1569 // Capture a frame and wait for it to synchronize with the encoder thread.
1570 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1571 WaitForEncodedFrame(2);
1572 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1573
1574 video_stream_encoder_->Stop();
1575}
1576
Sergey Silkin5ee69672019-07-02 14:18:34 +02001577TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001579 DataRate::BitsPerSec(kTargetBitrateBps),
1580 DataRate::BitsPerSec(kTargetBitrateBps),
1581 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001582
1583 VideoEncoderConfig video_encoder_config;
1584 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1585 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1586 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1587 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1588 kMaxPayloadLength);
1589
1590 // Capture a frame and wait for it to synchronize with the encoder thread.
1591 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1592 WaitForEncodedFrame(1);
1593 // The encoder will have been configured once when the first frame is
1594 // received.
1595 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1596 EXPECT_EQ(kTargetBitrateBps,
1597 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1598 EXPECT_EQ(kStartBitrateBps,
1599 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1600
Sergey Silkin6456e352019-07-08 17:56:40 +02001601 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1602 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001603 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1604 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1605 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1606 kMaxPayloadLength);
1607
1608 // Capture a frame and wait for it to synchronize with the encoder thread.
1609 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1610 WaitForEncodedFrame(2);
1611 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1612 // Bitrate limits have changed - rate allocator should be reconfigured,
1613 // encoder should not be reconfigured.
1614 EXPECT_EQ(kTargetBitrateBps * 2,
1615 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1616 EXPECT_EQ(kStartBitrateBps * 2,
1617 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1618 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1619
1620 video_stream_encoder_->Stop();
1621}
1622
Sergey Silkin6456e352019-07-08 17:56:40 +02001623TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001624 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001626 DataRate::BitsPerSec(kTargetBitrateBps),
1627 DataRate::BitsPerSec(kTargetBitrateBps),
1628 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001629
Sergey Silkincd02eba2020-01-20 14:48:40 +01001630 const uint32_t kMinEncBitrateKbps = 100;
1631 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001632 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001633 /*frame_size_pixels=*/codec_width_ * codec_height_,
1634 /*min_start_bitrate_bps=*/0,
1635 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1636 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001637 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1638
Sergey Silkincd02eba2020-01-20 14:48:40 +01001639 VideoEncoderConfig video_encoder_config;
1640 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1641 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1642 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1643 (kMinEncBitrateKbps + 1) * 1000;
1644 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1645 kMaxPayloadLength);
1646
1647 // When both encoder and app provide bitrate limits, the intersection of
1648 // provided sets should be used.
1649 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1650 WaitForEncodedFrame(1);
1651 EXPECT_EQ(kMaxEncBitrateKbps,
1652 bitrate_allocator_factory_.codec_config().maxBitrate);
1653 EXPECT_EQ(kMinEncBitrateKbps + 1,
1654 bitrate_allocator_factory_.codec_config().minBitrate);
1655
1656 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1657 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1658 (kMinEncBitrateKbps - 1) * 1000;
1659 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1660 kMaxPayloadLength);
1661 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001662 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001663 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001664 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001665 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001666 bitrate_allocator_factory_.codec_config().minBitrate);
1667
Sergey Silkincd02eba2020-01-20 14:48:40 +01001668 video_stream_encoder_->Stop();
1669}
1670
1671TEST_F(VideoStreamEncoderTest,
1672 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001674 DataRate::BitsPerSec(kTargetBitrateBps),
1675 DataRate::BitsPerSec(kTargetBitrateBps),
1676 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001677
1678 const uint32_t kMinAppBitrateKbps = 100;
1679 const uint32_t kMaxAppBitrateKbps = 200;
1680 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1681 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1682 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1683 /*frame_size_pixels=*/codec_width_ * codec_height_,
1684 /*min_start_bitrate_bps=*/0,
1685 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1686 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1687 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1688
1689 VideoEncoderConfig video_encoder_config;
1690 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1691 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1692 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1693 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001694 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1695 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001696
Sergey Silkincd02eba2020-01-20 14:48:40 +01001697 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1698 WaitForEncodedFrame(1);
1699 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001700 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001701 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001702 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001703
1704 video_stream_encoder_->Stop();
1705}
1706
1707TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001708 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001710 DataRate::BitsPerSec(kTargetBitrateBps),
1711 DataRate::BitsPerSec(kTargetBitrateBps),
1712 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001713
1714 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001715 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001716 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001717 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001718 fake_encoder_.SetResolutionBitrateLimits(
1719 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1720
1721 VideoEncoderConfig video_encoder_config;
1722 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1723 video_encoder_config.max_bitrate_bps = 0;
1724 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1725 kMaxPayloadLength);
1726
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001727 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001728 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1729 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001730 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1731 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001732 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1733 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1734
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001735 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001736 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1737 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001738 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1739 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001740 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1741 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1742
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001743 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001744 // encoder for 360p should be used.
1745 video_source_.IncomingCapturedFrame(
1746 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1747 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001748 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1749 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001750 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1751 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1752
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001753 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001754 // ignored.
1755 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1756 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001757 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1758 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001759 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1760 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001761 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1762 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001763 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1764 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1765
1766 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1767 // for 270p should be used.
1768 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1769 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001770 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1771 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001772 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1773 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1774
1775 video_stream_encoder_->Stop();
1776}
1777
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001778TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001780 DataRate::BitsPerSec(kTargetBitrateBps),
1781 DataRate::BitsPerSec(kTargetBitrateBps),
1782 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001783
1784 VideoEncoderConfig video_encoder_config;
1785 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1786 video_encoder_config.max_bitrate_bps = 0;
1787 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1788 kMaxPayloadLength);
1789
1790 // Encode 720p frame to get the default encoder target bitrate.
1791 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1792 WaitForEncodedFrame(1);
1793 const uint32_t kDefaultTargetBitrateFor720pKbps =
1794 bitrate_allocator_factory_.codec_config()
1795 .simulcastStream[0]
1796 .targetBitrate;
1797
1798 // Set the max recommended encoder bitrate to something lower than the default
1799 // target bitrate.
1800 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1801 1280 * 720, 10 * 1000, 10 * 1000,
1802 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1803 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1804
1805 // Change resolution to trigger encoder reinitialization.
1806 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1807 WaitForEncodedFrame(2);
1808 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1809 WaitForEncodedFrame(3);
1810
1811 // Ensure the target bitrate is capped by the max bitrate.
1812 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1813 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1814 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1815 .simulcastStream[0]
1816 .targetBitrate *
1817 1000,
1818 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1819
1820 video_stream_encoder_->Stop();
1821}
1822
mflodmancc3d4422017-08-03 08:27:51 -07001823TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001824 EXPECT_TRUE(video_source_.has_sinks());
1825 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001826 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001827 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001828 EXPECT_FALSE(video_source_.has_sinks());
1829 EXPECT_TRUE(new_video_source.has_sinks());
1830
mflodmancc3d4422017-08-03 08:27:51 -07001831 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001832}
1833
mflodmancc3d4422017-08-03 08:27:51 -07001834TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001835 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001836 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001837 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001838 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001839}
1840
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001841TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1842 constexpr int kRequestedResolutionAlignment = 7;
1843 video_source_.set_adaptation_enabled(true);
1844 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001845 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001846 DataRate::BitsPerSec(kTargetBitrateBps),
1847 DataRate::BitsPerSec(kTargetBitrateBps),
1848 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001849
1850 // On the 1st frame, we should have initialized the encoder and
1851 // asked for its resolution requirements.
1852 video_source_.IncomingCapturedFrame(
1853 CreateFrame(1, codec_width_, codec_height_));
1854 WaitForEncodedFrame(1);
1855 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1856 kRequestedResolutionAlignment);
1857
1858 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1859 // (It's up the to the encoder to potentially drop the previous frame,
1860 // to avoid coding back-to-back keyframes.)
1861 video_source_.IncomingCapturedFrame(
1862 CreateFrame(2, codec_width_, codec_height_));
1863 WaitForEncodedFrame(2);
1864 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1865
1866 video_stream_encoder_->Stop();
1867}
1868
Jonathan Yubc771b72017-12-08 17:04:29 -08001869TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1870 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001871 const int kWidth = 1280;
1872 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001873
1874 // We rely on the automatic resolution adaptation, but we handle framerate
1875 // adaptation manually by mocking the stats proxy.
1876 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001877
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001878 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001879 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001880 DataRate::BitsPerSec(kTargetBitrateBps),
1881 DataRate::BitsPerSec(kTargetBitrateBps),
1882 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001883 video_stream_encoder_->SetSource(&video_source_,
1884 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001885 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001886 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001887 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001888 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1889
Jonathan Yubc771b72017-12-08 17:04:29 -08001890 // Adapt down as far as possible.
1891 rtc::VideoSinkWants last_wants;
1892 int64_t t = 1;
1893 int loop_count = 0;
1894 do {
1895 ++loop_count;
1896 last_wants = video_source_.sink_wants();
1897
1898 // Simulate the framerate we've been asked to adapt to.
1899 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1900 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1901 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1902 mock_stats.input_frame_rate = fps;
1903 stats_proxy_->SetMockStats(mock_stats);
1904
1905 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1906 sink_.WaitForEncodedFrame(t);
1907 t += frame_interval_ms;
1908
mflodmancc3d4422017-08-03 08:27:51 -07001909 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001910 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001911 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001912 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1913 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001914 } while (video_source_.sink_wants().max_pixel_count <
1915 last_wants.max_pixel_count ||
1916 video_source_.sink_wants().max_framerate_fps <
1917 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001918
Jonathan Yubc771b72017-12-08 17:04:29 -08001919 // Verify that we've adapted all the way down.
1920 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001921 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001922 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1923 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001924 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001925 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1926 *video_source_.last_sent_height());
1927 EXPECT_EQ(kMinBalancedFramerateFps,
1928 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001929
Jonathan Yubc771b72017-12-08 17:04:29 -08001930 // Adapt back up the same number of times we adapted down.
1931 for (int i = 0; i < loop_count - 1; ++i) {
1932 last_wants = video_source_.sink_wants();
1933
1934 // Simulate the framerate we've been asked to adapt to.
1935 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1936 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1937 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1938 mock_stats.input_frame_rate = fps;
1939 stats_proxy_->SetMockStats(mock_stats);
1940
1941 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1942 sink_.WaitForEncodedFrame(t);
1943 t += frame_interval_ms;
1944
Henrik Boström91aa7322020-04-28 12:24:33 +02001945 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001946 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001947 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001948 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1949 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001950 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1951 last_wants.max_pixel_count ||
1952 video_source_.sink_wants().max_framerate_fps >
1953 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001954 }
1955
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001956 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001957 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001958 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001959 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1960 EXPECT_EQ((loop_count - 1) * 2,
1961 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001962
mflodmancc3d4422017-08-03 08:27:51 -07001963 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001964}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001965
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001966TEST_F(VideoStreamEncoderTest,
1967 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1968 video_stream_encoder_->OnBitrateUpdated(
1969 DataRate::BitsPerSec(kTargetBitrateBps),
1970 DataRate::BitsPerSec(kTargetBitrateBps),
1971 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001972 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001973
1974 const int kFrameWidth = 1280;
1975 const int kFrameHeight = 720;
1976
1977 int64_t ntp_time = kFrameIntervalMs;
1978
1979 // Force an input frame rate to be available, or the adaptation call won't
1980 // know what framerate to adapt form.
1981 const int kInputFps = 30;
1982 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1983 stats.input_frame_rate = kInputFps;
1984 stats_proxy_->SetMockStats(stats);
1985
1986 video_source_.set_adaptation_enabled(true);
1987 video_stream_encoder_->SetSource(
1988 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001989 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001990 video_source_.IncomingCapturedFrame(
1991 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1992 sink_.WaitForEncodedFrame(ntp_time);
1993 ntp_time += kFrameIntervalMs;
1994
1995 // Trigger CPU overuse.
1996 video_stream_encoder_->TriggerCpuOveruse();
1997 video_source_.IncomingCapturedFrame(
1998 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1999 sink_.WaitForEncodedFrame(ntp_time);
2000 ntp_time += kFrameIntervalMs;
2001
2002 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2003 EXPECT_EQ(std::numeric_limits<int>::max(),
2004 video_source_.sink_wants().max_pixel_count);
2005 // Some framerate constraint should be set.
2006 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2007 EXPECT_LT(restricted_fps, kInputFps);
2008 video_source_.IncomingCapturedFrame(
2009 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2010 sink_.WaitForEncodedFrame(ntp_time);
2011 ntp_time += 100;
2012
Henrik Boström2671dac2020-05-19 16:29:09 +02002013 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002014 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2015 // Give the encoder queue time to process the change in degradation preference
2016 // by waiting for an encoded frame.
2017 video_source_.IncomingCapturedFrame(
2018 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2019 sink_.WaitForEncodedFrame(ntp_time);
2020 ntp_time += kFrameIntervalMs;
2021
2022 video_stream_encoder_->TriggerQualityLow();
2023 video_source_.IncomingCapturedFrame(
2024 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2025 sink_.WaitForEncodedFrame(ntp_time);
2026 ntp_time += kFrameIntervalMs;
2027
2028 // Some resolution constraint should be set.
2029 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2030 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2031 kFrameWidth * kFrameHeight);
2032 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2033
2034 int pixel_count = video_source_.sink_wants().max_pixel_count;
2035 // Triggering a CPU underuse should not change the sink wants since it has
2036 // not been overused for resolution since we changed degradation preference.
2037 video_stream_encoder_->TriggerCpuUnderuse();
2038 video_source_.IncomingCapturedFrame(
2039 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2040 sink_.WaitForEncodedFrame(ntp_time);
2041 ntp_time += kFrameIntervalMs;
2042 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2043 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2044
2045 // Change the degradation preference back. CPU underuse should now adapt.
Henrik Boström2671dac2020-05-19 16:29:09 +02002046 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002047 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2048 video_source_.IncomingCapturedFrame(
2049 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2050 sink_.WaitForEncodedFrame(ntp_time);
2051 ntp_time += 100;
2052 // Resolution adaptations is gone after changing degradation preference.
2053 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2054 EXPECT_EQ(std::numeric_limits<int>::max(),
2055 video_source_.sink_wants().max_pixel_count);
2056 // The fps adaptation from above is now back.
2057 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2058
2059 // Trigger CPU underuse.
2060 video_stream_encoder_->TriggerCpuUnderuse();
2061 video_source_.IncomingCapturedFrame(
2062 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2063 sink_.WaitForEncodedFrame(ntp_time);
2064 ntp_time += kFrameIntervalMs;
2065 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2066
2067 video_stream_encoder_->Stop();
2068}
2069
mflodmancc3d4422017-08-03 08:27:51 -07002070TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002071 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002072 DataRate::BitsPerSec(kTargetBitrateBps),
2073 DataRate::BitsPerSec(kTargetBitrateBps),
2074 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002075 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002076
sprangc5d62e22017-04-02 23:53:04 -07002077 const int kFrameWidth = 1280;
2078 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002079
Åsa Persson8c1bf952018-09-13 10:42:19 +02002080 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002081
kthelgason5e13d412016-12-01 03:59:51 -08002082 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002083 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002084 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002085 frame_timestamp += kFrameIntervalMs;
2086
perkj803d97f2016-11-01 11:45:46 -07002087 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002088 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002089 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002090 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002091 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002092 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002093
asapersson0944a802017-04-07 00:57:58 -07002094 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002095 // wanted resolution.
2096 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2097 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2098 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002099 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002100
2101 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002102 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002103 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002104 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002105 // Give the encoder queue time to process the change in degradation preference
2106 // by waiting for an encoded frame.
2107 new_video_source.IncomingCapturedFrame(
2108 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2109 sink_.WaitForEncodedFrame(frame_timestamp);
2110 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002111 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002112 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002113
sprangc5d62e22017-04-02 23:53:04 -07002114 // Force an input frame rate to be available, or the adaptation call won't
2115 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002116 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002117 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002118 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002119 stats_proxy_->SetMockStats(stats);
2120
mflodmancc3d4422017-08-03 08:27:51 -07002121 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002122 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002123 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002124 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002125 frame_timestamp += kFrameIntervalMs;
2126
2127 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002128 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002129 EXPECT_EQ(std::numeric_limits<int>::max(),
2130 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002131 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002132
asapersson02465b82017-04-10 01:12:52 -07002133 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002134 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2135 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002136 // Give the encoder queue time to process the change in degradation preference
2137 // by waiting for an encoded frame.
2138 new_video_source.IncomingCapturedFrame(
2139 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2140 sink_.WaitForEncodedFrame(frame_timestamp);
2141 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002142 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002143
mflodmancc3d4422017-08-03 08:27:51 -07002144 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002145 new_video_source.IncomingCapturedFrame(
2146 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002147 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002148 frame_timestamp += kFrameIntervalMs;
2149
2150 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002151 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002152
2153 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002154 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002155 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002156 // Give the encoder queue time to process the change in degradation preference
2157 // by waiting for an encoded frame.
2158 new_video_source.IncomingCapturedFrame(
2159 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2160 sink_.WaitForEncodedFrame(frame_timestamp);
2161 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002162 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2163 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002164 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002165 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002166
2167 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002168 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002169 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002170 // Give the encoder queue time to process the change in degradation preference
2171 // by waiting for an encoded frame.
2172 new_video_source.IncomingCapturedFrame(
2173 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2174 sink_.WaitForEncodedFrame(frame_timestamp);
2175 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002176 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2177 EXPECT_EQ(std::numeric_limits<int>::max(),
2178 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002179 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002180
mflodmancc3d4422017-08-03 08:27:51 -07002181 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002182}
2183
mflodmancc3d4422017-08-03 08:27:51 -07002184TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002185 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002186 DataRate::BitsPerSec(kTargetBitrateBps),
2187 DataRate::BitsPerSec(kTargetBitrateBps),
2188 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002189
asaperssonfab67072017-04-04 05:51:49 -07002190 const int kWidth = 1280;
2191 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002192 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002193 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002194 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2195 EXPECT_FALSE(stats.bw_limited_resolution);
2196 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2197
2198 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002199 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002200 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002201 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002202
2203 stats = stats_proxy_->GetStats();
2204 EXPECT_TRUE(stats.bw_limited_resolution);
2205 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2206
2207 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002208 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002209 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002210 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002211
2212 stats = stats_proxy_->GetStats();
2213 EXPECT_FALSE(stats.bw_limited_resolution);
2214 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2215 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2216
mflodmancc3d4422017-08-03 08:27:51 -07002217 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002218}
2219
mflodmancc3d4422017-08-03 08:27:51 -07002220TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002222 DataRate::BitsPerSec(kTargetBitrateBps),
2223 DataRate::BitsPerSec(kTargetBitrateBps),
2224 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002225
2226 const int kWidth = 1280;
2227 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002228 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002229 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002230 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2231 EXPECT_FALSE(stats.cpu_limited_resolution);
2232 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2233
2234 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002235 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002236 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002237 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002238
2239 stats = stats_proxy_->GetStats();
2240 EXPECT_TRUE(stats.cpu_limited_resolution);
2241 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2242
2243 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002244 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002245 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002246 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002247
2248 stats = stats_proxy_->GetStats();
2249 EXPECT_FALSE(stats.cpu_limited_resolution);
2250 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002251 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002252
mflodmancc3d4422017-08-03 08:27:51 -07002253 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002254}
2255
mflodmancc3d4422017-08-03 08:27:51 -07002256TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002257 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002258 DataRate::BitsPerSec(kTargetBitrateBps),
2259 DataRate::BitsPerSec(kTargetBitrateBps),
2260 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002261
asaperssonfab67072017-04-04 05:51:49 -07002262 const int kWidth = 1280;
2263 const int kHeight = 720;
2264 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002265 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002266 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002267 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002268 EXPECT_FALSE(stats.cpu_limited_resolution);
2269 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2270
asaperssonfab67072017-04-04 05:51:49 -07002271 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002272 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002273 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002274 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002275 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002276 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002277 EXPECT_TRUE(stats.cpu_limited_resolution);
2278 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2279
2280 // Set new source with adaptation still enabled.
2281 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002282 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002283 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002284
asaperssonfab67072017-04-04 05:51:49 -07002285 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002286 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002287 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002288 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002289 EXPECT_TRUE(stats.cpu_limited_resolution);
2290 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2291
2292 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002293 video_stream_encoder_->SetSource(&new_video_source,
2294 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002295
asaperssonfab67072017-04-04 05:51:49 -07002296 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002297 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002298 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002299 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002300 EXPECT_FALSE(stats.cpu_limited_resolution);
2301 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2302
2303 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002304 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002305 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002306
asaperssonfab67072017-04-04 05:51:49 -07002307 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002308 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002309 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002310 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002311 EXPECT_TRUE(stats.cpu_limited_resolution);
2312 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2313
asaperssonfab67072017-04-04 05:51:49 -07002314 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002315 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002316 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002317 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002318 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002319 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002320 EXPECT_FALSE(stats.cpu_limited_resolution);
2321 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002322 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002323
mflodmancc3d4422017-08-03 08:27:51 -07002324 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002325}
2326
mflodmancc3d4422017-08-03 08:27:51 -07002327TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002329 DataRate::BitsPerSec(kTargetBitrateBps),
2330 DataRate::BitsPerSec(kTargetBitrateBps),
2331 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002332
asaperssonfab67072017-04-04 05:51:49 -07002333 const int kWidth = 1280;
2334 const int kHeight = 720;
2335 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002336 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002337 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002338 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002339 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002340 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002341
2342 // Set new source with adaptation still enabled.
2343 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002344 video_stream_encoder_->SetSource(&new_video_source,
2345 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002346
asaperssonfab67072017-04-04 05:51:49 -07002347 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002348 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002349 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002350 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002351 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002352 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002353
asaperssonfab67072017-04-04 05:51:49 -07002354 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002355 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002356 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002357 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002358 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002359 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002360 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002361 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002362
asaperssonfab67072017-04-04 05:51:49 -07002363 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002364 video_stream_encoder_->SetSource(&new_video_source,
2365 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002366
asaperssonfab67072017-04-04 05:51:49 -07002367 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002368 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002369 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002370 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002371 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002372 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002373
asapersson02465b82017-04-10 01:12:52 -07002374 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002375 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002376 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002377
asaperssonfab67072017-04-04 05:51:49 -07002378 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002379 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002380 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002381 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002382 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002383 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2384 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002385
mflodmancc3d4422017-08-03 08:27:51 -07002386 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002387}
2388
mflodmancc3d4422017-08-03 08:27:51 -07002389TEST_F(VideoStreamEncoderTest,
2390 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002391 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002392 DataRate::BitsPerSec(kTargetBitrateBps),
2393 DataRate::BitsPerSec(kTargetBitrateBps),
2394 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002395
2396 const int kWidth = 1280;
2397 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002398 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002399 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002400 video_source_.IncomingCapturedFrame(
2401 CreateFrame(timestamp_ms, kWidth, kHeight));
2402 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002403 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2404 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2406
2407 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002408 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002409 timestamp_ms += kFrameIntervalMs;
2410 video_source_.IncomingCapturedFrame(
2411 CreateFrame(timestamp_ms, kWidth, kHeight));
2412 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002413 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2415 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2416
2417 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002418 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002419 timestamp_ms += kFrameIntervalMs;
2420 video_source_.IncomingCapturedFrame(
2421 CreateFrame(timestamp_ms, kWidth, kHeight));
2422 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002423 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2426
Niels Möller4db138e2018-04-19 09:04:13 +02002427 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002428 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002429
2430 VideoEncoderConfig video_encoder_config;
2431 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2432 // Make format different, to force recreation of encoder.
2433 video_encoder_config.video_format.parameters["foo"] = "foo";
2434 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002435 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002436 timestamp_ms += kFrameIntervalMs;
2437 video_source_.IncomingCapturedFrame(
2438 CreateFrame(timestamp_ms, kWidth, kHeight));
2439 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002440 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2441 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2442 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2443
mflodmancc3d4422017-08-03 08:27:51 -07002444 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002445}
2446
mflodmancc3d4422017-08-03 08:27:51 -07002447TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002448 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002450 DataRate::BitsPerSec(kTargetBitrateBps),
2451 DataRate::BitsPerSec(kTargetBitrateBps),
2452 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2453
2454 const int kWidth = 1280;
2455 const int kHeight = 720;
2456 int sequence = 1;
2457
2458 // Enable BALANCED preference, no initial limitation.
2459 test::FrameForwarder source;
2460 video_stream_encoder_->SetSource(&source,
2461 webrtc::DegradationPreference::BALANCED);
2462 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2463 WaitForEncodedFrame(sequence++);
2464 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2465 EXPECT_FALSE(stats.cpu_limited_resolution);
2466 EXPECT_FALSE(stats.cpu_limited_framerate);
2467 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2468
2469 // Trigger CPU overuse, should now adapt down.
2470 video_stream_encoder_->TriggerCpuOveruse();
2471 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2472 WaitForEncodedFrame(sequence++);
2473 stats = stats_proxy_->GetStats();
2474 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2475
2476 // Set new degradation preference should clear restrictions since we changed
2477 // from BALANCED.
2478 video_stream_encoder_->SetSource(
2479 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2480 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2481 WaitForEncodedFrame(sequence++);
2482 stats = stats_proxy_->GetStats();
2483 EXPECT_FALSE(stats.cpu_limited_resolution);
2484 EXPECT_FALSE(stats.cpu_limited_framerate);
2485 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2486
2487 // Force an input frame rate to be available, or the adaptation call won't
2488 // know what framerate to adapt from.
2489 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2490 mock_stats.input_frame_rate = 30;
2491 stats_proxy_->SetMockStats(mock_stats);
2492 video_stream_encoder_->TriggerCpuOveruse();
2493 stats_proxy_->ResetMockStats();
2494 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2495 WaitForEncodedFrame(sequence++);
2496
2497 // We have now adapted once.
2498 stats = stats_proxy_->GetStats();
2499 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2500
2501 // Back to BALANCED, should clear the restrictions again.
2502 video_stream_encoder_->SetSource(&source,
2503 webrtc::DegradationPreference::BALANCED);
2504 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2505 WaitForEncodedFrame(sequence++);
2506 stats = stats_proxy_->GetStats();
2507 EXPECT_FALSE(stats.cpu_limited_resolution);
2508 EXPECT_FALSE(stats.cpu_limited_framerate);
2509 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2510
2511 video_stream_encoder_->Stop();
2512}
2513
2514TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002515 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002517 DataRate::BitsPerSec(kTargetBitrateBps),
2518 DataRate::BitsPerSec(kTargetBitrateBps),
2519 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002520
asapersson0944a802017-04-07 00:57:58 -07002521 const int kWidth = 1280;
2522 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002523 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002524
asaperssonfab67072017-04-04 05:51:49 -07002525 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002526 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002527 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002528 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002529 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002530 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2531
asapersson02465b82017-04-10 01:12:52 -07002532 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002533 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002534 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002535 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002536 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002537 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002538 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002539 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2540
2541 // Set new source with adaptation still enabled.
2542 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002543 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002544 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002545
2546 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002547 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002548 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002549 stats = stats_proxy_->GetStats();
2550 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002551 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002552 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2553
sprangc5d62e22017-04-02 23:53:04 -07002554 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002555 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002556 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002557 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002558 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002559 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002560 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002561 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002562 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002563 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002564 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2565
sprangc5d62e22017-04-02 23:53:04 -07002566 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002567 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002568 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2569 mock_stats.input_frame_rate = 30;
2570 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002571 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002572 stats_proxy_->ResetMockStats();
2573
2574 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002575 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002576 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002577
2578 // Framerate now adapted.
2579 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002580 EXPECT_FALSE(stats.cpu_limited_resolution);
2581 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002582 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2583
2584 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002585 video_stream_encoder_->SetSource(&new_video_source,
2586 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002587 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002588 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002589 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002590
2591 stats = stats_proxy_->GetStats();
2592 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002593 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002594 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2595
2596 // Try to trigger overuse. Should not succeed.
2597 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002598 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002599 stats_proxy_->ResetMockStats();
2600
2601 stats = stats_proxy_->GetStats();
2602 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002603 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002604 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2605
2606 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002607 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002608 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002609 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002610 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002611 stats = stats_proxy_->GetStats();
2612 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002613 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002614 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002615
2616 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002617 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002618 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002619 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002620 stats = stats_proxy_->GetStats();
2621 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002622 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002623 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2624
2625 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002626 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002627 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002628 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002629 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002630 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002631 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002632 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002633 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002634 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002635 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2636
2637 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002638 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002639 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002640 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002641 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002642 stats = stats_proxy_->GetStats();
2643 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002644 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002645 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002646 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002647
mflodmancc3d4422017-08-03 08:27:51 -07002648 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002649}
2650
mflodmancc3d4422017-08-03 08:27:51 -07002651TEST_F(VideoStreamEncoderTest,
2652 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002653 const int kWidth = 1280;
2654 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002655 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002656 DataRate::BitsPerSec(kTargetBitrateBps),
2657 DataRate::BitsPerSec(kTargetBitrateBps),
2658 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002659
asaperssonfab67072017-04-04 05:51:49 -07002660 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002661 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002662
asaperssonfab67072017-04-04 05:51:49 -07002663 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002664 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002665
asaperssonfab67072017-04-04 05:51:49 -07002666 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002667 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002668
asaperssonfab67072017-04-04 05:51:49 -07002669 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002670 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002671
kthelgason876222f2016-11-29 01:44:11 -08002672 // Expect a scale down.
2673 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002674 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002675
asapersson02465b82017-04-10 01:12:52 -07002676 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002677 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002678 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002679 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002680
asaperssonfab67072017-04-04 05:51:49 -07002681 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002682 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002683 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002684 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002685
asaperssonfab67072017-04-04 05:51:49 -07002686 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002687 EXPECT_EQ(std::numeric_limits<int>::max(),
2688 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002689
asaperssonfab67072017-04-04 05:51:49 -07002690 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002691 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002692 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002693 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002694
asapersson02465b82017-04-10 01:12:52 -07002695 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002696 EXPECT_EQ(std::numeric_limits<int>::max(),
2697 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002698
mflodmancc3d4422017-08-03 08:27:51 -07002699 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002700}
2701
mflodmancc3d4422017-08-03 08:27:51 -07002702TEST_F(VideoStreamEncoderTest,
2703 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002704 const int kWidth = 1280;
2705 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002706 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002707 DataRate::BitsPerSec(kTargetBitrateBps),
2708 DataRate::BitsPerSec(kTargetBitrateBps),
2709 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002710
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002711 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002712 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002713 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002714 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002715
2716 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002717 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002719 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2721
2722 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002723 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002724 EXPECT_THAT(source.sink_wants(),
2725 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002726 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2727 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2728 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2729
2730 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002732 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2733 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2734 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2735
mflodmancc3d4422017-08-03 08:27:51 -07002736 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002737}
2738
mflodmancc3d4422017-08-03 08:27:51 -07002739TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002740 const int kWidth = 1280;
2741 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002742 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002743 DataRate::BitsPerSec(kTargetBitrateBps),
2744 DataRate::BitsPerSec(kTargetBitrateBps),
2745 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002746
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002747 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002748 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002749 video_stream_encoder_->SetSource(&source,
2750 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002751 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2752 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002753 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002754
2755 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002756 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002757 EXPECT_THAT(source.sink_wants(),
2758 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002759 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2760 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2761 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2762
2763 // Trigger adapt down for same input resolution, expect no change.
2764 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2765 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002766 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002767 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2768 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2769 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2770
2771 // Trigger adapt down for larger input resolution, expect no change.
2772 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2773 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002774 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002775 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2776 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2777 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2778
mflodmancc3d4422017-08-03 08:27:51 -07002779 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002780}
2781
mflodmancc3d4422017-08-03 08:27:51 -07002782TEST_F(VideoStreamEncoderTest,
2783 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002784 const int kWidth = 1280;
2785 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002787 DataRate::BitsPerSec(kTargetBitrateBps),
2788 DataRate::BitsPerSec(kTargetBitrateBps),
2789 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002790
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002791 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002792 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002793 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002794 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002795
2796 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002797 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2801
2802 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002803 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002805 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2806 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2807
mflodmancc3d4422017-08-03 08:27:51 -07002808 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002809}
2810
mflodmancc3d4422017-08-03 08:27:51 -07002811TEST_F(VideoStreamEncoderTest,
2812 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002813 const int kWidth = 1280;
2814 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002816 DataRate::BitsPerSec(kTargetBitrateBps),
2817 DataRate::BitsPerSec(kTargetBitrateBps),
2818 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002819
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002820 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002821 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002822 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002823 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002824
2825 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002826 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002827 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002828 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2830
2831 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002832 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002833 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002834 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002835 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2836
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002838}
2839
mflodmancc3d4422017-08-03 08:27:51 -07002840TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002841 const int kWidth = 1280;
2842 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002843 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002844 DataRate::BitsPerSec(kTargetBitrateBps),
2845 DataRate::BitsPerSec(kTargetBitrateBps),
2846 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002847
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002848 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002849 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002850 video_stream_encoder_->SetSource(&source,
2851 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002852
2853 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2854 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002855 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002856 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2857 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2858 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2859
2860 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002861 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002862 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002863 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2864 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2865 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2866
mflodmancc3d4422017-08-03 08:27:51 -07002867 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002868}
2869
mflodmancc3d4422017-08-03 08:27:51 -07002870TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002871 const int kWidth = 1280;
2872 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002874 DataRate::BitsPerSec(kTargetBitrateBps),
2875 DataRate::BitsPerSec(kTargetBitrateBps),
2876 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002877
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002878 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002879 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002880 video_stream_encoder_->SetSource(&source,
2881 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002882
2883 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2884 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002885 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002886 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2887 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2888 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2889
2890 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002891 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002892 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002893 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2894 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2895 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2896
mflodmancc3d4422017-08-03 08:27:51 -07002897 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002898}
2899
mflodmancc3d4422017-08-03 08:27:51 -07002900TEST_F(VideoStreamEncoderTest,
2901 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002902 const int kWidth = 1280;
2903 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002904 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002905 DataRate::BitsPerSec(kTargetBitrateBps),
2906 DataRate::BitsPerSec(kTargetBitrateBps),
2907 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002908
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002909 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002910 AdaptingFrameForwarder source;
2911 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002912 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002913 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002914
2915 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002916 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002917 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002918 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2919 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2920
2921 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002922 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002923 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002924 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002925 EXPECT_THAT(source.sink_wants(),
2926 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002927 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2928 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2929
2930 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002931 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002932 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002933 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2934 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2935 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2936
mflodmancc3d4422017-08-03 08:27:51 -07002937 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002938}
2939
mflodmancc3d4422017-08-03 08:27:51 -07002940TEST_F(VideoStreamEncoderTest,
2941 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002942 const int kWidth = 1280;
2943 const int kHeight = 720;
2944 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002946 DataRate::BitsPerSec(kTargetBitrateBps),
2947 DataRate::BitsPerSec(kTargetBitrateBps),
2948 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002949
2950 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2951 stats.input_frame_rate = kInputFps;
2952 stats_proxy_->SetMockStats(stats);
2953
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002954 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002955 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2956 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002957 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002958
2959 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002960 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002961 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2962 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002963 EXPECT_THAT(video_source_.sink_wants(),
2964 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002965
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002966 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002967 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002968 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002969 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002970 // Give the encoder queue time to process the change in degradation preference
2971 // by waiting for an encoded frame.
2972 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2973 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002974 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002975
2976 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002977 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002978 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2979 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002980 EXPECT_THAT(new_video_source.sink_wants(),
2981 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002982
2983 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002985 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002986
mflodmancc3d4422017-08-03 08:27:51 -07002987 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002988}
2989
mflodmancc3d4422017-08-03 08:27:51 -07002990TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002991 const int kWidth = 1280;
2992 const int kHeight = 720;
2993 const size_t kNumFrames = 10;
2994
Henrik Boström381d1092020-05-12 18:49:07 +02002995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002996 DataRate::BitsPerSec(kTargetBitrateBps),
2997 DataRate::BitsPerSec(kTargetBitrateBps),
2998 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002999
asaperssond0de2952017-04-21 01:47:31 -07003000 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003001 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003002 video_source_.set_adaptation_enabled(true);
3003
3004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3005 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3006
3007 int downscales = 0;
3008 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003009 video_source_.IncomingCapturedFrame(
3010 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3011 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003012
asaperssonfab67072017-04-04 05:51:49 -07003013 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003014 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003015 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003016 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003017
3018 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3019 ++downscales;
3020
3021 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3022 EXPECT_EQ(downscales,
3023 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3024 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003025 }
mflodmancc3d4422017-08-03 08:27:51 -07003026 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003027}
3028
mflodmancc3d4422017-08-03 08:27:51 -07003029TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003030 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3031 const int kWidth = 1280;
3032 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003034 DataRate::BitsPerSec(kTargetBitrateBps),
3035 DataRate::BitsPerSec(kTargetBitrateBps),
3036 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003037
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003038 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003039 AdaptingFrameForwarder source;
3040 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003042 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003043
Åsa Persson8c1bf952018-09-13 10:42:19 +02003044 int64_t timestamp_ms = kFrameIntervalMs;
3045 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003046 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003047 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003048 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3049 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3050
3051 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003052 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003053 timestamp_ms += kFrameIntervalMs;
3054 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3055 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003056 EXPECT_THAT(source.sink_wants(),
3057 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003058 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3059 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3060
3061 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003062 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003063 timestamp_ms += kFrameIntervalMs;
3064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003065 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003066 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003067 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3068 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3069
3070 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003071 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003072 timestamp_ms += kFrameIntervalMs;
3073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3074 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003075 EXPECT_THAT(source.sink_wants(),
3076 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003077 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3078 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3079
3080 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003081 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003082 timestamp_ms += kFrameIntervalMs;
3083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003084 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003085 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003086 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3087 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3088
mflodmancc3d4422017-08-03 08:27:51 -07003089 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003090}
3091
mflodmancc3d4422017-08-03 08:27:51 -07003092TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003093 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3094 const int kWidth = 1280;
3095 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003096 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003097 DataRate::BitsPerSec(kTargetBitrateBps),
3098 DataRate::BitsPerSec(kTargetBitrateBps),
3099 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003100
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003101 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003102 AdaptingFrameForwarder source;
3103 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003104 video_stream_encoder_->SetSource(&source,
3105 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003106
Åsa Persson8c1bf952018-09-13 10:42:19 +02003107 int64_t timestamp_ms = kFrameIntervalMs;
3108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003109 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003110 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003111 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3112 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3113
3114 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003116 timestamp_ms += kFrameIntervalMs;
3117 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3118 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003119 EXPECT_THAT(source.sink_wants(),
3120 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003121 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3122 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3123
3124 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003125 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003126 timestamp_ms += kFrameIntervalMs;
3127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003128 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003129 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003130 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3131 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3132
3133 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003134 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003135 timestamp_ms += kFrameIntervalMs;
3136 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3137 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003138 EXPECT_THAT(source.sink_wants(),
3139 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003140 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3141 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3142
3143 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003144 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003145 timestamp_ms += kFrameIntervalMs;
3146 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003147 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003148 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003149 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3150 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3151
mflodmancc3d4422017-08-03 08:27:51 -07003152 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003153}
3154
Sergey Silkin41c650b2019-10-14 13:12:19 +02003155TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3156 fake_encoder_.SetResolutionBitrateLimits(
3157 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3158
Henrik Boström381d1092020-05-12 18:49:07 +02003159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003160 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3161 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3162 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3163 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003164
3165 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3166 AdaptingFrameForwarder source;
3167 source.set_adaptation_enabled(true);
3168 video_stream_encoder_->SetSource(
3169 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3170
3171 // Insert 720p frame.
3172 int64_t timestamp_ms = kFrameIntervalMs;
3173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3174 WaitForEncodedFrame(1280, 720);
3175
3176 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003177 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003178 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3179 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3180 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3181 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003182 video_stream_encoder_->TriggerQualityLow();
3183
3184 // Insert 720p frame. It should be downscaled and encoded.
3185 timestamp_ms += kFrameIntervalMs;
3186 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3187 WaitForEncodedFrame(960, 540);
3188
3189 // Trigger adapt up. Higher resolution should not be requested duo to lack
3190 // of bitrate.
3191 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003192 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003193
3194 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003196 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3197 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3198 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3199 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003200
3201 // Trigger adapt up. Higher resolution should be requested.
3202 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003203 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003204
3205 video_stream_encoder_->Stop();
3206}
3207
3208TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3209 fake_encoder_.SetResolutionBitrateLimits(
3210 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3211
3212 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003213 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003214 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3215 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3216 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3217 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003218
3219 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3220 AdaptingFrameForwarder source;
3221 source.set_adaptation_enabled(true);
3222 video_stream_encoder_->SetSource(
3223 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3224
3225 // Insert 720p frame. It should be dropped and lower resolution should be
3226 // requested.
3227 int64_t timestamp_ms = kFrameIntervalMs;
3228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3229 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003230 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003231
3232 // Insert 720p frame. It should be downscaled and encoded.
3233 timestamp_ms += kFrameIntervalMs;
3234 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3235 WaitForEncodedFrame(960, 540);
3236
3237 video_stream_encoder_->Stop();
3238}
3239
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003240class BalancedDegradationTest : public VideoStreamEncoderTest {
3241 protected:
3242 void SetupTest() {
3243 // Reset encoder for field trials to take effect.
3244 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003245 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003246
3247 // Enable BALANCED preference.
3248 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003249 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3250 }
3251
3252 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003254 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3255 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003256 }
3257
Åsa Persson45b176f2019-09-30 11:19:05 +02003258 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003259 timestamp_ms_ += kFrameIntervalMs;
3260 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003261 }
3262
3263 void InsertFrameAndWaitForEncoded() {
3264 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003265 sink_.WaitForEncodedFrame(timestamp_ms_);
3266 }
3267
3268 const int kWidth = 640; // pixels:640x360=230400
3269 const int kHeight = 360;
3270 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3271 int64_t timestamp_ms_ = 0;
3272 AdaptingFrameForwarder source_;
3273};
3274
3275TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3276 test::ScopedFieldTrials field_trials(
3277 "WebRTC-Video-BalancedDegradationSettings/"
3278 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3279 SetupTest();
3280
3281 // Force input frame rate.
3282 const int kInputFps = 24;
3283 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3284 stats.input_frame_rate = kInputFps;
3285 stats_proxy_->SetMockStats(stats);
3286
Åsa Persson45b176f2019-09-30 11:19:05 +02003287 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003288 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003289
3290 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003291 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3292 // QP samples.
3293 EXPECT_FALSE(
3294 video_stream_encoder_
3295 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003296 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003297
3298 video_stream_encoder_->Stop();
3299}
3300
3301TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3302 test::ScopedFieldTrials field_trials(
3303 "WebRTC-Video-BalancedDegradationSettings/"
3304 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3305 SetupTest();
3306
3307 // Force input frame rate.
3308 const int kInputFps = 25;
3309 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3310 stats.input_frame_rate = kInputFps;
3311 stats_proxy_->SetMockStats(stats);
3312
Åsa Persson45b176f2019-09-30 11:19:05 +02003313 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003314 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003315
3316 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003317 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3318 // samples.
3319 EXPECT_TRUE(
3320 video_stream_encoder_
3321 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003323
3324 video_stream_encoder_->Stop();
3325}
3326
3327TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3328 test::ScopedFieldTrials field_trials(
3329 "WebRTC-Video-BalancedDegradationSettings/"
3330 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3331 SetupTest();
3332
3333 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3334
Åsa Persson45b176f2019-09-30 11:19:05 +02003335 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003336 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003337
3338 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3339 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003340 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003341
3342 video_stream_encoder_->Stop();
3343}
3344
Åsa Perssonccfb3402019-09-25 15:13:04 +02003345TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003346 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003347 "WebRTC-Video-BalancedDegradationSettings/"
3348 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003349 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003350
Åsa Persson1b247f12019-08-14 17:26:39 +02003351 const int kMinBitrateBps = 425000;
3352 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003353 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003354
Åsa Persson45b176f2019-09-30 11:19:05 +02003355 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003356 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003357 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3358
3359 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3360 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003361 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003362 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003363 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3364
3365 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3366 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003367 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003368 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003369 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3370
Åsa Persson30ab0152019-08-27 12:22:33 +02003371 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3372 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003373 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003374 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
3375 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003376 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3377
3378 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003379 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003380 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003381 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003382
Åsa Persson30ab0152019-08-27 12:22:33 +02003383 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003384 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003385 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003386 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003387 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003388 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3389
3390 video_stream_encoder_->Stop();
3391}
3392
Åsa Perssonccfb3402019-09-25 15:13:04 +02003393TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003394 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3395 test::ScopedFieldTrials field_trials(
3396 "WebRTC-Video-BalancedDegradationSettings/"
3397 "pixels:57600|129600|230400,fps:7|24|24/");
3398 SetupTest();
3399 OnBitrateUpdated(kLowTargetBitrateBps);
3400
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003401 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003402
3403 // Insert frame, expect scaled down:
3404 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3405 InsertFrame();
3406 EXPECT_FALSE(WaitForFrame(1000));
3407 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3408 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3409
3410 // Insert frame, expect scaled down:
3411 // resolution (320x180@24fps).
3412 InsertFrame();
3413 EXPECT_FALSE(WaitForFrame(1000));
3414 EXPECT_LT(source_.sink_wants().max_pixel_count,
3415 source_.last_wants().max_pixel_count);
3416 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3417
3418 // Frame should not be dropped (min pixels per frame reached).
3419 InsertFrameAndWaitForEncoded();
3420
3421 video_stream_encoder_->Stop();
3422}
3423
3424TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003425 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003426 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003427 "WebRTC-Video-BalancedDegradationSettings/"
3428 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003429 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003430
Åsa Persson30ab0152019-08-27 12:22:33 +02003431 const int kResolutionMinBitrateBps = 435000;
3432 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003433 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003434
Åsa Persson45b176f2019-09-30 11:19:05 +02003435 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003436 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003437 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3438
3439 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3440 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003441 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003442 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003443 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3444
3445 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3446 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003447 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003448 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003449 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3450
3451 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3452 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003453 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003454 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson1b247f12019-08-14 17:26:39 +02003455 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3456
Åsa Persson30ab0152019-08-27 12:22:33 +02003457 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3458 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003459 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003460 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003461 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3462
3463 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3464 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003465 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003466 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3467
3468 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003469 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003470 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003471 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003472 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003473 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3474
3475 video_stream_encoder_->Stop();
3476}
3477
Åsa Perssonccfb3402019-09-25 15:13:04 +02003478TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003479 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003480 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003481 "WebRTC-Video-BalancedDegradationSettings/"
3482 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003483 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003484
Åsa Persson30ab0152019-08-27 12:22:33 +02003485 const int kMinBitrateBps = 425000;
3486 const int kTooLowMinBitrateBps = 424000;
3487 const int kResolutionMinBitrateBps = 435000;
3488 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003489 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003490
Åsa Persson45b176f2019-09-30 11:19:05 +02003491 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003492 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003493 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3494
3495 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3496 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003497 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003498 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003499 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3500
3501 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3502 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003503 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003504 VerifyFpsEqResolutionLt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003505 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3506
3507 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3508 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003509 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003510 VerifyFpsLtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003511 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3512
3513 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3514 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003515 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003516 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3517
3518 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003519 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003520 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003521 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003522 VerifyFpsGtResolutionEq(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003523 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3524
3525 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003526 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003527 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003528 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003529 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3530
3531 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003532 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003533 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003534 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003535 VerifyFpsEqResolutionGt(source_.sink_wants(), source_.last_wants());
Åsa Persson30ab0152019-08-27 12:22:33 +02003536 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3537
Åsa Persson1b247f12019-08-14 17:26:39 +02003538 video_stream_encoder_->Stop();
3539}
3540
mflodmancc3d4422017-08-03 08:27:51 -07003541TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003542 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3543 const int kWidth = 1280;
3544 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003546 DataRate::BitsPerSec(kTargetBitrateBps),
3547 DataRate::BitsPerSec(kTargetBitrateBps),
3548 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003549
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003550 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003551 AdaptingFrameForwarder source;
3552 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003553 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003554 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003555
Åsa Persson8c1bf952018-09-13 10:42:19 +02003556 int64_t timestamp_ms = kFrameIntervalMs;
3557 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003558 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003559 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003560 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3561 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3563 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564
3565 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003566 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003567 timestamp_ms += kFrameIntervalMs;
3568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3569 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003570 EXPECT_THAT(source.sink_wants(),
3571 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003572 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3574 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3575 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3576
3577 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003578 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003579 timestamp_ms += kFrameIntervalMs;
3580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3581 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003582 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003583 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3584 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3585 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3586 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587
Jonathan Yubc771b72017-12-08 17:04:29 -08003588 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003589 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3592 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003593 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003594 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003596 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003597 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3598
Jonathan Yubc771b72017-12-08 17:04:29 -08003599 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003600 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003601 timestamp_ms += kFrameIntervalMs;
3602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3603 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003604 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003605 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003606 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3608 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3609 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3610
Jonathan Yubc771b72017-12-08 17:04:29 -08003611 // Trigger quality adapt down, expect no change (min resolution reached).
3612 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003613 timestamp_ms += kFrameIntervalMs;
3614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3615 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003616 VerifyFpsMaxResolutionEq(source.sink_wants(), last_wants);
3617 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3619 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3620 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3621
3622 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003623 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003624 timestamp_ms += kFrameIntervalMs;
3625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3626 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003627 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Jonathan Yubc771b72017-12-08 17:04:29 -08003628 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3630 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3631 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3632
3633 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003634 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003635 timestamp_ms += kFrameIntervalMs;
3636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3637 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003638 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
3639 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3641 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3642 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3643
3644 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003645 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003646 timestamp_ms += kFrameIntervalMs;
3647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3648 WaitForEncodedFrame(timestamp_ms);
Jonathan Yubc771b72017-12-08 17:04:29 -08003649 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
asaperssond0de2952017-04-21 01:47:31 -07003650 last_wants = source.sink_wants();
3651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3652 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003653 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003654 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3655
3656 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003657 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003658 timestamp_ms += kFrameIntervalMs;
3659 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3660 WaitForEncodedFrame(timestamp_ms);
asapersson09f05612017-05-15 23:40:18 -07003661 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
asaperssond0de2952017-04-21 01:47:31 -07003662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3663 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003664 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003665 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3666
3667 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003668 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003669 timestamp_ms += kFrameIntervalMs;
3670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003671 WaitForEncodedFrame(kWidth, kHeight);
asapersson09f05612017-05-15 23:40:18 -07003672 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003673 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003676 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003677 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003678
mflodmancc3d4422017-08-03 08:27:51 -07003679 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003680}
3681
mflodmancc3d4422017-08-03 08:27:51 -07003682TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003683 const int kWidth = 640;
3684 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003685
Henrik Boström381d1092020-05-12 18:49:07 +02003686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003687 DataRate::BitsPerSec(kTargetBitrateBps),
3688 DataRate::BitsPerSec(kTargetBitrateBps),
3689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003690
perkj803d97f2016-11-01 11:45:46 -07003691 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003692 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003693 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003694 }
3695
mflodmancc3d4422017-08-03 08:27:51 -07003696 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003697 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003698 video_source_.IncomingCapturedFrame(CreateFrame(
3699 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003700 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003701 }
3702
mflodmancc3d4422017-08-03 08:27:51 -07003703 video_stream_encoder_->Stop();
3704 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003705 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003706
Ying Wangef3998f2019-12-09 13:06:53 +01003707 EXPECT_METRIC_EQ(
3708 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3709 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003710 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3711}
3712
mflodmancc3d4422017-08-03 08:27:51 -07003713TEST_F(VideoStreamEncoderTest,
3714 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003716 DataRate::BitsPerSec(kTargetBitrateBps),
3717 DataRate::BitsPerSec(kTargetBitrateBps),
3718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003719 const int kWidth = 640;
3720 const int kHeight = 360;
3721
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003722 video_stream_encoder_->SetSource(&video_source_,
3723 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003724
3725 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3726 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003727 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003728 }
3729
mflodmancc3d4422017-08-03 08:27:51 -07003730 video_stream_encoder_->Stop();
3731 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003732 stats_proxy_.reset();
3733
3734 EXPECT_EQ(0,
3735 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3736}
3737
mflodmancc3d4422017-08-03 08:27:51 -07003738TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003739 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003740 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003741
3742 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003743 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003744 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003745 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3746 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003747
sprang57c2fff2017-01-16 06:24:02 -08003748 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003749 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003750 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003751 DataRate::BitsPerSec(kLowTargetBitrateBps),
3752 DataRate::BitsPerSec(kLowTargetBitrateBps),
3753 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003754
sprang57c2fff2017-01-16 06:24:02 -08003755 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003756 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3757 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003758 VideoBitrateAllocation bitrate_allocation =
3759 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003760 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003761 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003762 // TODO(srte): The use of millisecs here looks like an error, but the tests
3763 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003764 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003765
3766 // Not called on second frame.
3767 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3768 .Times(0);
3769 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003770 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3771 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003772 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003773
3774 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003775 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3776 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003777 const int64_t start_time_ms = rtc::TimeMillis();
3778 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3779 video_source_.IncomingCapturedFrame(
3780 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3781 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003782 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003783 }
3784
3785 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003786 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003787
mflodmancc3d4422017-08-03 08:27:51 -07003788 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003789}
3790
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003791TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3792 // 2 TLs configured, temporal layers supported by encoder.
3793 const int kNumTemporalLayers = 2;
3794 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3795 fake_encoder_.SetTemporalLayersSupported(0, true);
3796
3797 // Bitrate allocated across temporal layers.
3798 const int kTl0Bps = kTargetBitrateBps *
3799 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003800 kNumTemporalLayers, /*temporal_id*/ 0,
3801 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003802 const int kTl1Bps = kTargetBitrateBps *
3803 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003804 kNumTemporalLayers, /*temporal_id*/ 1,
3805 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003806 VideoBitrateAllocation expected_bitrate;
3807 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3808 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3809
3810 VerifyAllocatedBitrate(expected_bitrate);
3811 video_stream_encoder_->Stop();
3812}
3813
3814TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3815 // 2 TLs configured, temporal layers not supported by encoder.
3816 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3817 fake_encoder_.SetTemporalLayersSupported(0, false);
3818
3819 // Temporal layers not supported by the encoder.
3820 // Total bitrate should be at ti:0.
3821 VideoBitrateAllocation expected_bitrate;
3822 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3823
3824 VerifyAllocatedBitrate(expected_bitrate);
3825 video_stream_encoder_->Stop();
3826}
3827
3828TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3829 // 2 TLs configured, temporal layers only supported for first stream.
3830 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3831 fake_encoder_.SetTemporalLayersSupported(0, true);
3832 fake_encoder_.SetTemporalLayersSupported(1, false);
3833
3834 const int kS0Bps = 150000;
3835 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003836 kS0Bps *
3837 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3838 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003839 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003840 kS0Bps *
3841 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3842 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003843 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3844 // Temporal layers not supported by si:1.
3845 VideoBitrateAllocation expected_bitrate;
3846 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3847 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3848 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3849
3850 VerifyAllocatedBitrate(expected_bitrate);
3851 video_stream_encoder_->Stop();
3852}
3853
Niels Möller7dc26b72017-12-06 10:27:48 +01003854TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3855 const int kFrameWidth = 1280;
3856 const int kFrameHeight = 720;
3857 const int kFramerate = 24;
3858
Henrik Boström381d1092020-05-12 18:49:07 +02003859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003860 DataRate::BitsPerSec(kTargetBitrateBps),
3861 DataRate::BitsPerSec(kTargetBitrateBps),
3862 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003863 test::FrameForwarder source;
3864 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003865 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003866
3867 // Insert a single frame, triggering initial configuration.
3868 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3869 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3870
3871 EXPECT_EQ(
3872 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3873 kDefaultFramerate);
3874
3875 // Trigger reconfigure encoder (without resetting the entire instance).
3876 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003877 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003878 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3879 video_encoder_config.number_of_streams = 1;
3880 video_encoder_config.video_stream_factory =
3881 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3882 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003883 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003884 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3885
3886 // Detector should be updated with fps limit from codec config.
3887 EXPECT_EQ(
3888 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3889 kFramerate);
3890
3891 // Trigger overuse, max framerate should be reduced.
3892 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3893 stats.input_frame_rate = kFramerate;
3894 stats_proxy_->SetMockStats(stats);
3895 video_stream_encoder_->TriggerCpuOveruse();
3896 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3897 int adapted_framerate =
3898 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3899 EXPECT_LT(adapted_framerate, kFramerate);
3900
3901 // Trigger underuse, max framerate should go back to codec configured fps.
3902 // Set extra low fps, to make sure it's actually reset, not just incremented.
3903 stats = stats_proxy_->GetStats();
3904 stats.input_frame_rate = adapted_framerate / 2;
3905 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003906 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003907 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3908 EXPECT_EQ(
3909 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3910 kFramerate);
3911
3912 video_stream_encoder_->Stop();
3913}
3914
3915TEST_F(VideoStreamEncoderTest,
3916 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3917 const int kFrameWidth = 1280;
3918 const int kFrameHeight = 720;
3919 const int kLowFramerate = 15;
3920 const int kHighFramerate = 25;
3921
Henrik Boström381d1092020-05-12 18:49:07 +02003922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003923 DataRate::BitsPerSec(kTargetBitrateBps),
3924 DataRate::BitsPerSec(kTargetBitrateBps),
3925 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003926 test::FrameForwarder source;
3927 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003928 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003929
3930 // Trigger initial configuration.
3931 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003932 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003933 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3934 video_encoder_config.number_of_streams = 1;
3935 video_encoder_config.video_stream_factory =
3936 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3937 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3938 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003939 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003940 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3941
3942 EXPECT_EQ(
3943 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3944 kLowFramerate);
3945
3946 // Trigger overuse, max framerate should be reduced.
3947 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3948 stats.input_frame_rate = kLowFramerate;
3949 stats_proxy_->SetMockStats(stats);
3950 video_stream_encoder_->TriggerCpuOveruse();
3951 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3952 int adapted_framerate =
3953 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3954 EXPECT_LT(adapted_framerate, kLowFramerate);
3955
3956 // Reconfigure the encoder with a new (higher max framerate), max fps should
3957 // still respect the adaptation.
3958 video_encoder_config.video_stream_factory =
3959 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3960 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3961 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003962 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003963 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3964
3965 EXPECT_EQ(
3966 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3967 adapted_framerate);
3968
3969 // Trigger underuse, max framerate should go back to codec configured fps.
3970 stats = stats_proxy_->GetStats();
3971 stats.input_frame_rate = adapted_framerate;
3972 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003973 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003974 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3975 EXPECT_EQ(
3976 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3977 kHighFramerate);
3978
3979 video_stream_encoder_->Stop();
3980}
3981
mflodmancc3d4422017-08-03 08:27:51 -07003982TEST_F(VideoStreamEncoderTest,
3983 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003984 const int kFrameWidth = 1280;
3985 const int kFrameHeight = 720;
3986 const int kFramerate = 24;
3987
Henrik Boström381d1092020-05-12 18:49:07 +02003988 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003989 DataRate::BitsPerSec(kTargetBitrateBps),
3990 DataRate::BitsPerSec(kTargetBitrateBps),
3991 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003992 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003993 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003994 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003995
3996 // Trigger initial configuration.
3997 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003998 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003999 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4000 video_encoder_config.number_of_streams = 1;
4001 video_encoder_config.video_stream_factory =
4002 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
4003 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004004 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004005 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004006 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004007
Niels Möller7dc26b72017-12-06 10:27:48 +01004008 EXPECT_EQ(
4009 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4010 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004011
4012 // Trigger overuse, max framerate should be reduced.
4013 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4014 stats.input_frame_rate = kFramerate;
4015 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004016 video_stream_encoder_->TriggerCpuOveruse();
4017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004018 int adapted_framerate =
4019 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004020 EXPECT_LT(adapted_framerate, kFramerate);
4021
4022 // Change degradation preference to not enable framerate scaling. Target
4023 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004024 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004025 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004026 EXPECT_EQ(
4027 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4028 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004029
mflodmancc3d4422017-08-03 08:27:51 -07004030 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004031}
4032
mflodmancc3d4422017-08-03 08:27:51 -07004033TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004034 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004036 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4037 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4038 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004039 const int kWidth = 640;
4040 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004041
asaperssonfab67072017-04-04 05:51:49 -07004042 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004043
4044 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004045 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004046
4047 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004048 EXPECT_TRUE_WAIT(
4049 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004050
sprangc5d62e22017-04-02 23:53:04 -07004051 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004052
asaperssonfab67072017-04-04 05:51:49 -07004053 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004054 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004055 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004056
4057 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004058 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004059
Henrik Boström2671dac2020-05-19 16:29:09 +02004060 EXPECT_TRUE_WAIT(
4061 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004062
mflodmancc3d4422017-08-03 08:27:51 -07004063 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004064}
4065
mflodmancc3d4422017-08-03 08:27:51 -07004066TEST_F(VideoStreamEncoderTest,
4067 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004068 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004069 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004070 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4071 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4072 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004073 const int kWidth = 640;
4074 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004075
4076 // We expect the n initial frames to get dropped.
4077 int i;
4078 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004079 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004080 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004081 }
4082 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004083 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004084 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004085
4086 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004087 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004088
mflodmancc3d4422017-08-03 08:27:51 -07004089 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004090}
4091
mflodmancc3d4422017-08-03 08:27:51 -07004092TEST_F(VideoStreamEncoderTest,
4093 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004094 const int kWidth = 640;
4095 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004096 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004097 DataRate::BitsPerSec(kLowTargetBitrateBps),
4098 DataRate::BitsPerSec(kLowTargetBitrateBps),
4099 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004100
4101 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004102 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004103 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004104
asaperssonfab67072017-04-04 05:51:49 -07004105 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004106 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004107 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004108
mflodmancc3d4422017-08-03 08:27:51 -07004109 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004110}
4111
mflodmancc3d4422017-08-03 08:27:51 -07004112TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004113 const int kWidth = 640;
4114 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004115 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004116
4117 VideoEncoderConfig video_encoder_config;
4118 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4119 // Make format different, to force recreation of encoder.
4120 video_encoder_config.video_format.parameters["foo"] = "foo";
4121 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004122 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004124 DataRate::BitsPerSec(kLowTargetBitrateBps),
4125 DataRate::BitsPerSec(kLowTargetBitrateBps),
4126 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004127
kthelgasonb83797b2017-02-14 11:57:25 -08004128 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004129 video_stream_encoder_->SetSource(&video_source_,
4130 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004131
asaperssonfab67072017-04-04 05:51:49 -07004132 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004133 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004134 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004135
mflodmancc3d4422017-08-03 08:27:51 -07004136 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004137 fake_encoder_.SetQualityScaling(true);
4138}
4139
Åsa Persson139f4dc2019-08-02 09:29:58 +02004140TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4141 webrtc::test::ScopedFieldTrials field_trials(
4142 "WebRTC-Video-QualityScalerSettings/"
4143 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4144 // Reset encoder for field trials to take effect.
4145 ConfigureEncoder(video_encoder_config_.Copy());
4146 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4147 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4148 const int kWidth = 640;
4149 const int kHeight = 360;
4150
Henrik Boström381d1092020-05-12 18:49:07 +02004151 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004152 DataRate::BitsPerSec(kTargetBitrateBps),
4153 DataRate::BitsPerSec(kTargetBitrateBps),
4154 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004155 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4156 // Frame should not be dropped.
4157 WaitForEncodedFrame(1);
4158
Henrik Boström381d1092020-05-12 18:49:07 +02004159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004160 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4161 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4162 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004163 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4164 // Frame should not be dropped.
4165 WaitForEncodedFrame(2);
4166
Henrik Boström381d1092020-05-12 18:49:07 +02004167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004168 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4169 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4170 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004171 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4172 // Expect to drop this frame, the wait should time out.
4173 ExpectDroppedFrame();
4174
4175 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004176 EXPECT_TRUE_WAIT(
4177 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004178 video_stream_encoder_->Stop();
4179}
4180
Åsa Perssone644a032019-11-08 15:56:00 +01004181TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4182 webrtc::test::ScopedFieldTrials field_trials(
4183 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4184
4185 // Reset encoder for field trials to take effect.
4186 VideoEncoderConfig config = video_encoder_config_.Copy();
4187 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004188 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004189 ConfigureEncoder(std::move(config));
4190 fake_encoder_.SetQp(kQpLow);
4191
4192 // Enable MAINTAIN_FRAMERATE preference.
4193 AdaptingFrameForwarder source;
4194 source.set_adaptation_enabled(true);
4195 video_stream_encoder_->SetSource(&source,
4196 DegradationPreference::MAINTAIN_FRAMERATE);
4197
4198 // Start at low bitrate.
4199 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004200 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4201 DataRate::BitsPerSec(kLowBitrateBps),
4202 DataRate::BitsPerSec(kLowBitrateBps),
4203 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004204
4205 // Expect first frame to be dropped and resolution to be limited.
4206 const int kWidth = 1280;
4207 const int kHeight = 720;
4208 const int64_t kFrameIntervalMs = 100;
4209 int64_t timestamp_ms = kFrameIntervalMs;
4210 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4211 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004212 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4213 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004214
4215 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004216 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4217 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004218
4219 // Insert frames and advance |min_duration_ms|.
4220 for (size_t i = 1; i <= 10; i++) {
4221 timestamp_ms += kFrameIntervalMs;
4222 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4223 WaitForEncodedFrame(timestamp_ms);
4224 }
4225 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4226 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4227
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004228 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004229
4230 // Insert frame should trigger high BW and release quality limitation.
4231 timestamp_ms += kFrameIntervalMs;
4232 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4233 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004234 // The ramp-up code involves the adaptation queue, give it time to execute.
4235 // TODO(hbos): Can we await an appropriate event instead?
4236 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004237 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004238
4239 // Frame should not be adapted.
4240 timestamp_ms += kFrameIntervalMs;
4241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4242 WaitForEncodedFrame(kWidth, kHeight);
4243 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4244
4245 video_stream_encoder_->Stop();
4246}
4247
mflodmancc3d4422017-08-03 08:27:51 -07004248TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004249 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4250 const int kTooSmallWidth = 10;
4251 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004252 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004253 DataRate::BitsPerSec(kTargetBitrateBps),
4254 DataRate::BitsPerSec(kTargetBitrateBps),
4255 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004256
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004257 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004258 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004259 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004260 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004261 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004262 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4263
4264 // Trigger adapt down, too small frame, expect no change.
4265 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004266 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004267 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004268 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004269 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4270 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4271
mflodmancc3d4422017-08-03 08:27:51 -07004272 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004273}
4274
mflodmancc3d4422017-08-03 08:27:51 -07004275TEST_F(VideoStreamEncoderTest,
4276 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004277 const int kTooSmallWidth = 10;
4278 const int kTooSmallHeight = 10;
4279 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004280 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004281 DataRate::BitsPerSec(kTargetBitrateBps),
4282 DataRate::BitsPerSec(kTargetBitrateBps),
4283 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004284
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004285 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004286 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004287 video_stream_encoder_->SetSource(&source,
4288 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004289 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004290 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4291 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4292
4293 // Trigger adapt down, expect limited framerate.
4294 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004295 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004296 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004297 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4299 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4300 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4301
4302 // Trigger adapt down, too small frame, expect no change.
4303 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004304 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004305 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004306 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4308 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4310
mflodmancc3d4422017-08-03 08:27:51 -07004311 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004312}
4313
mflodmancc3d4422017-08-03 08:27:51 -07004314TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004315 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004316 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004317 DataRate::BitsPerSec(kTargetBitrateBps),
4318 DataRate::BitsPerSec(kTargetBitrateBps),
4319 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004320 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004321 const int kFrameWidth = 1280;
4322 const int kFrameHeight = 720;
4323 video_source_.IncomingCapturedFrame(
4324 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004325 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004326 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004327}
4328
sprangb1ca0732017-02-01 08:38:12 -08004329// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004330TEST_F(VideoStreamEncoderTest,
4331 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004333 DataRate::BitsPerSec(kTargetBitrateBps),
4334 DataRate::BitsPerSec(kTargetBitrateBps),
4335 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004336
4337 const int kFrameWidth = 1280;
4338 const int kFrameHeight = 720;
4339 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004340 // requested by
4341 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004342 video_source_.set_adaptation_enabled(true);
4343
4344 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004345 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004346 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004347
4348 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004349 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004350 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004351 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004352 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004353
asaperssonfab67072017-04-04 05:51:49 -07004354 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004355 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004356 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004357 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004358 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004359
mflodmancc3d4422017-08-03 08:27:51 -07004360 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004361}
sprangfe627f32017-03-29 08:24:59 -07004362
mflodmancc3d4422017-08-03 08:27:51 -07004363TEST_F(VideoStreamEncoderTest,
4364 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004365 const int kFrameWidth = 1280;
4366 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004367
Henrik Boström381d1092020-05-12 18:49:07 +02004368 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004369 DataRate::BitsPerSec(kTargetBitrateBps),
4370 DataRate::BitsPerSec(kTargetBitrateBps),
4371 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004373 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004374 video_source_.set_adaptation_enabled(true);
4375
sprang4847ae62017-06-27 07:06:52 -07004376 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004377
4378 video_source_.IncomingCapturedFrame(
4379 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004380 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004381
4382 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004383 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004384
4385 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004386 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004388 video_source_.IncomingCapturedFrame(
4389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004390 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004391 }
4392
4393 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004394 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004395 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004396 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004397 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004398 video_source_.IncomingCapturedFrame(
4399 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004400 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004401 ++num_frames_dropped;
4402 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004403 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004404 }
4405 }
4406
sprang4847ae62017-06-27 07:06:52 -07004407 // Add some slack to account for frames dropped by the frame dropper.
4408 const int kErrorMargin = 1;
4409 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004410 kErrorMargin);
4411
4412 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004413 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004414 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004415 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004416 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004417 video_source_.IncomingCapturedFrame(
4418 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004419 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004420 ++num_frames_dropped;
4421 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004422 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004423 }
4424 }
sprang4847ae62017-06-27 07:06:52 -07004425 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004426 kErrorMargin);
4427
4428 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004429 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004430 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004431 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004432 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004433 video_source_.IncomingCapturedFrame(
4434 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004435 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004436 ++num_frames_dropped;
4437 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004438 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004439 }
4440 }
sprang4847ae62017-06-27 07:06:52 -07004441 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004442 kErrorMargin);
4443
4444 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004445 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004446 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004447 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004448 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004449 video_source_.IncomingCapturedFrame(
4450 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004451 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004452 ++num_frames_dropped;
4453 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004454 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004455 }
4456 }
4457 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4458
mflodmancc3d4422017-08-03 08:27:51 -07004459 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004460}
4461
mflodmancc3d4422017-08-03 08:27:51 -07004462TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004463 const int kFramerateFps = 5;
4464 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004465 const int kFrameWidth = 1280;
4466 const int kFrameHeight = 720;
4467
sprang4847ae62017-06-27 07:06:52 -07004468 // Reconfigure encoder with two temporal layers and screensharing, which will
4469 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004470 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004471
Henrik Boström381d1092020-05-12 18:49:07 +02004472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004473 DataRate::BitsPerSec(kTargetBitrateBps),
4474 DataRate::BitsPerSec(kTargetBitrateBps),
4475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004476 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004477 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004478 video_source_.set_adaptation_enabled(true);
4479
sprang4847ae62017-06-27 07:06:52 -07004480 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004481
4482 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004483 rtc::VideoSinkWants last_wants;
4484 do {
4485 last_wants = video_source_.sink_wants();
4486
sprangc5d62e22017-04-02 23:53:04 -07004487 // Insert frames to get a new fps estimate...
4488 for (int j = 0; j < kFramerateFps; ++j) {
4489 video_source_.IncomingCapturedFrame(
4490 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004491 if (video_source_.last_sent_width()) {
4492 sink_.WaitForEncodedFrame(timestamp_ms);
4493 }
sprangc5d62e22017-04-02 23:53:04 -07004494 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004495 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004496 }
4497 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004498 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004499 } while (video_source_.sink_wants().max_framerate_fps <
4500 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004501
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004502 EXPECT_THAT(video_source_.sink_wants(),
4503 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004504
mflodmancc3d4422017-08-03 08:27:51 -07004505 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004506}
asaperssonf7e294d2017-06-13 23:25:22 -07004507
mflodmancc3d4422017-08-03 08:27:51 -07004508TEST_F(VideoStreamEncoderTest,
4509 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004510 const int kWidth = 1280;
4511 const int kHeight = 720;
4512 const int64_t kFrameIntervalMs = 150;
4513 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004515 DataRate::BitsPerSec(kTargetBitrateBps),
4516 DataRate::BitsPerSec(kTargetBitrateBps),
4517 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004518
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004519 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004520 AdaptingFrameForwarder source;
4521 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004522 video_stream_encoder_->SetSource(&source,
4523 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004524 timestamp_ms += kFrameIntervalMs;
4525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004526 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004527 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004528 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4530 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4531
4532 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004533 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004534 timestamp_ms += kFrameIntervalMs;
4535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004536 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004537 EXPECT_THAT(source.sink_wants(),
4538 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4541 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4542
4543 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004544 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004545 timestamp_ms += kFrameIntervalMs;
4546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004547 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004548 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4551 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4552
4553 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004554 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004555 timestamp_ms += kFrameIntervalMs;
4556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004557 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004558 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4561 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4562
4563 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004564 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004565 timestamp_ms += kFrameIntervalMs;
4566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004567 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004568 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4571 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4572
4573 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004574 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004575 timestamp_ms += kFrameIntervalMs;
4576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004577 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004578 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4581 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4582
4583 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004584 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004585 timestamp_ms += kFrameIntervalMs;
4586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004587 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004588 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4589 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4591 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4592
4593 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004594 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004595 timestamp_ms += kFrameIntervalMs;
4596 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004597 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004598 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4599 rtc::VideoSinkWants last_wants = source.sink_wants();
4600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4602 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4603
4604 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004605 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004606 timestamp_ms += kFrameIntervalMs;
4607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004608 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004609 VerifyFpsEqResolutionEq(source.sink_wants(), last_wants);
4610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4612 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4613
4614 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004615 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004616 timestamp_ms += kFrameIntervalMs;
4617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004618 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004619 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4622 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4623
4624 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004625 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004626 timestamp_ms += kFrameIntervalMs;
4627 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004628 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004629 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4632 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4633
4634 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004635 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004636 timestamp_ms += kFrameIntervalMs;
4637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004638 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004639 VerifyFpsGtResolutionEq(source.sink_wants(), source.last_wants());
4640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4642 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4643
4644 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004645 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004646 timestamp_ms += kFrameIntervalMs;
4647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004648 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004649 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4650 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4651 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4652 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4653
4654 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004655 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004656 timestamp_ms += kFrameIntervalMs;
4657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004658 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004659 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4661 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4662 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4663
4664 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004665 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004666 timestamp_ms += kFrameIntervalMs;
4667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004668 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004669 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4672 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4673
Åsa Persson30ab0152019-08-27 12:22:33 +02004674 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004675 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004676 timestamp_ms += kFrameIntervalMs;
4677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004678 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004679 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004680 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004681 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4682 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4683 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4684
4685 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004686 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004687 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004688 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4689
mflodmancc3d4422017-08-03 08:27:51 -07004690 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004691}
4692
mflodmancc3d4422017-08-03 08:27:51 -07004693TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004694 const int kWidth = 1280;
4695 const int kHeight = 720;
4696 const int64_t kFrameIntervalMs = 150;
4697 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004699 DataRate::BitsPerSec(kTargetBitrateBps),
4700 DataRate::BitsPerSec(kTargetBitrateBps),
4701 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004702
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004703 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004704 AdaptingFrameForwarder source;
4705 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004706 video_stream_encoder_->SetSource(&source,
4707 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004708 timestamp_ms += kFrameIntervalMs;
4709 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004710 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004711 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004712 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4713 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4714 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4715 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4716 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4718
4719 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004720 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004721 timestamp_ms += kFrameIntervalMs;
4722 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004723 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004724 EXPECT_THAT(source.sink_wants(),
4725 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004726 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4727 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4728 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4730 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4731 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4732
4733 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004734 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004735 timestamp_ms += kFrameIntervalMs;
4736 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004737 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004738 VerifyFpsMaxResolutionLt(source.sink_wants(), source.last_wants());
4739 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4740 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4741 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4742 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4743 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4744 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4745
4746 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004747 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004748 timestamp_ms += kFrameIntervalMs;
4749 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004750 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004751 VerifyFpsLtResolutionEq(source.sink_wants(), source.last_wants());
4752 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4753 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4754 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4755 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4756 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4757 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4758
4759 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004760 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004761 timestamp_ms += kFrameIntervalMs;
4762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004763 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004764 VerifyFpsMaxResolutionEq(source.sink_wants(), source.last_wants());
4765 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4766 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4767 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4768 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4769 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4770 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4771
4772 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004773 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004774 timestamp_ms += kFrameIntervalMs;
4775 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004776 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004777 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
4778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4779 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4780 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4781 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4782 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4783 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4784
4785 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004786 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004787 timestamp_ms += kFrameIntervalMs;
4788 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004789 WaitForEncodedFrame(kWidth, kHeight);
asaperssonf7e294d2017-06-13 23:25:22 -07004790 VerifyFpsMaxResolutionGt(source.sink_wants(), source.last_wants());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004791 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004792 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4793 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4794 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4795 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4796 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4797 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4798
4799 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004800 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004801 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004802 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4803 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4804
mflodmancc3d4422017-08-03 08:27:51 -07004805 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004806}
4807
mflodmancc3d4422017-08-03 08:27:51 -07004808TEST_F(VideoStreamEncoderTest,
4809 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004810 const int kWidth = 640;
4811 const int kHeight = 360;
4812 const int kFpsLimit = 15;
4813 const int64_t kFrameIntervalMs = 150;
4814 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004816 DataRate::BitsPerSec(kTargetBitrateBps),
4817 DataRate::BitsPerSec(kTargetBitrateBps),
4818 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004819
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004820 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004821 AdaptingFrameForwarder source;
4822 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004823 video_stream_encoder_->SetSource(&source,
4824 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004825 timestamp_ms += kFrameIntervalMs;
4826 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004827 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004828 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4830 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4831 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4832 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4833 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4834 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4835
4836 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004837 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004838 timestamp_ms += kFrameIntervalMs;
4839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004840 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004841 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4845 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4846 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4847 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4848
4849 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004850 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004851 timestamp_ms += kFrameIntervalMs;
4852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004853 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004854 VerifyFpsEqResolutionLt(source.sink_wants(), source.last_wants());
4855 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4856 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4857 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4858 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4859 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4861
4862 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004863 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004864 timestamp_ms += kFrameIntervalMs;
4865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004866 WaitForEncodedFrame(timestamp_ms);
asaperssonf7e294d2017-06-13 23:25:22 -07004867 VerifyFpsEqResolutionGt(source.sink_wants(), source.last_wants());
4868 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4870 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4872 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4874
4875 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004876 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004877 timestamp_ms += kFrameIntervalMs;
4878 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004879 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004880 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004881 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4882 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4883 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4884 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4885 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4886 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4887
4888 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004889 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004890 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004891 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4892 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4893
mflodmancc3d4422017-08-03 08:27:51 -07004894 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004895}
4896
mflodmancc3d4422017-08-03 08:27:51 -07004897TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004898 const int kFrameWidth = 1920;
4899 const int kFrameHeight = 1080;
4900 // 3/4 of 1920.
4901 const int kAdaptedFrameWidth = 1440;
4902 // 3/4 of 1080 rounded down to multiple of 4.
4903 const int kAdaptedFrameHeight = 808;
4904 const int kFramerate = 24;
4905
Henrik Boström381d1092020-05-12 18:49:07 +02004906 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004907 DataRate::BitsPerSec(kTargetBitrateBps),
4908 DataRate::BitsPerSec(kTargetBitrateBps),
4909 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004910 // Trigger reconfigure encoder (without resetting the entire instance).
4911 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004912 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004913 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4914 video_encoder_config.number_of_streams = 1;
4915 video_encoder_config.video_stream_factory =
4916 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004917 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004918 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004919 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004920
4921 video_source_.set_adaptation_enabled(true);
4922
4923 video_source_.IncomingCapturedFrame(
4924 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004925 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004926
4927 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004928 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004929 video_source_.IncomingCapturedFrame(
4930 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004931 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004932
mflodmancc3d4422017-08-03 08:27:51 -07004933 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004934}
4935
mflodmancc3d4422017-08-03 08:27:51 -07004936TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004937 const int kFrameWidth = 1280;
4938 const int kFrameHeight = 720;
4939 const int kLowFps = 2;
4940 const int kHighFps = 30;
4941
Henrik Boström381d1092020-05-12 18:49:07 +02004942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004943 DataRate::BitsPerSec(kTargetBitrateBps),
4944 DataRate::BitsPerSec(kTargetBitrateBps),
4945 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004946
4947 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4948 max_framerate_ = kLowFps;
4949
4950 // Insert 2 seconds of 2fps video.
4951 for (int i = 0; i < kLowFps * 2; ++i) {
4952 video_source_.IncomingCapturedFrame(
4953 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4954 WaitForEncodedFrame(timestamp_ms);
4955 timestamp_ms += 1000 / kLowFps;
4956 }
4957
4958 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004959 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004960 DataRate::BitsPerSec(kTargetBitrateBps),
4961 DataRate::BitsPerSec(kTargetBitrateBps),
4962 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004963 video_source_.IncomingCapturedFrame(
4964 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4965 WaitForEncodedFrame(timestamp_ms);
4966 timestamp_ms += 1000 / kLowFps;
4967
4968 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4969
4970 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004971 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004972 const int kFrameIntervalMs = 1000 / kHighFps;
4973 max_framerate_ = kHighFps;
4974 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4975 video_source_.IncomingCapturedFrame(
4976 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4977 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4978 // be dropped if the encoder hans't been updated with the new higher target
4979 // framerate yet, causing it to overshoot the target bitrate and then
4980 // suffering the wrath of the media optimizer.
4981 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4982 timestamp_ms += kFrameIntervalMs;
4983 }
4984
4985 // Don expect correct measurement just yet, but it should be higher than
4986 // before.
4987 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4988
mflodmancc3d4422017-08-03 08:27:51 -07004989 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004990}
4991
mflodmancc3d4422017-08-03 08:27:51 -07004992TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004993 const int kFrameWidth = 1280;
4994 const int kFrameHeight = 720;
4995 const int kTargetBitrateBps = 1000000;
4996
4997 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004998 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02004999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005000 DataRate::BitsPerSec(kTargetBitrateBps),
5001 DataRate::BitsPerSec(kTargetBitrateBps),
5002 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005003 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005004
5005 // Insert a first video frame, causes another bitrate update.
5006 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5007 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5008 video_source_.IncomingCapturedFrame(
5009 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5010 WaitForEncodedFrame(timestamp_ms);
5011
5012 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5014 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5015 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005016
5017 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005018 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005019 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005020
5021 // Bitrate observer should not be called.
5022 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5023 video_source_.IncomingCapturedFrame(
5024 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5025 ExpectDroppedFrame();
5026
mflodmancc3d4422017-08-03 08:27:51 -07005027 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005028}
ilnik6b826ef2017-06-16 06:53:48 -07005029
Niels Möller4db138e2018-04-19 09:04:13 +02005030TEST_F(VideoStreamEncoderTest,
5031 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5032 const int kFrameWidth = 1280;
5033 const int kFrameHeight = 720;
5034 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005036 DataRate::BitsPerSec(kTargetBitrateBps),
5037 DataRate::BitsPerSec(kTargetBitrateBps),
5038 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005039 video_source_.IncomingCapturedFrame(
5040 CreateFrame(1, kFrameWidth, kFrameHeight));
5041 WaitForEncodedFrame(1);
5042 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5043 .low_encode_usage_threshold_percent,
5044 default_options.low_encode_usage_threshold_percent);
5045 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5046 .high_encode_usage_threshold_percent,
5047 default_options.high_encode_usage_threshold_percent);
5048 video_stream_encoder_->Stop();
5049}
5050
5051TEST_F(VideoStreamEncoderTest,
5052 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5053 const int kFrameWidth = 1280;
5054 const int kFrameHeight = 720;
5055 CpuOveruseOptions hardware_options;
5056 hardware_options.low_encode_usage_threshold_percent = 150;
5057 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005058 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005059
Henrik Boström381d1092020-05-12 18:49:07 +02005060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005061 DataRate::BitsPerSec(kTargetBitrateBps),
5062 DataRate::BitsPerSec(kTargetBitrateBps),
5063 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005064 video_source_.IncomingCapturedFrame(
5065 CreateFrame(1, kFrameWidth, kFrameHeight));
5066 WaitForEncodedFrame(1);
5067 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5068 .low_encode_usage_threshold_percent,
5069 hardware_options.low_encode_usage_threshold_percent);
5070 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5071 .high_encode_usage_threshold_percent,
5072 hardware_options.high_encode_usage_threshold_percent);
5073 video_stream_encoder_->Stop();
5074}
5075
Niels Möller6bb5ab92019-01-11 11:11:10 +01005076TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5077 const int kFrameWidth = 320;
5078 const int kFrameHeight = 240;
5079 const int kFps = 30;
5080 const int kTargetBitrateBps = 120000;
5081 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5082
Henrik Boström381d1092020-05-12 18:49:07 +02005083 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005084 DataRate::BitsPerSec(kTargetBitrateBps),
5085 DataRate::BitsPerSec(kTargetBitrateBps),
5086 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005087
5088 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5089 max_framerate_ = kFps;
5090
5091 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5092 fake_encoder_.SimulateOvershoot(1.0);
5093 int num_dropped = 0;
5094 for (int i = 0; i < kNumFramesInRun; ++i) {
5095 video_source_.IncomingCapturedFrame(
5096 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5097 // Wait up to two frame durations for a frame to arrive.
5098 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5099 ++num_dropped;
5100 }
5101 timestamp_ms += 1000 / kFps;
5102 }
5103
Erik Språnga8d48ab2019-02-08 14:17:40 +01005104 // Framerate should be measured to be near the expected target rate.
5105 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5106
5107 // Frame drops should be within 5% of expected 0%.
5108 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005109
5110 // Make encoder produce frames at double the expected bitrate during 3 seconds
5111 // of video, verify number of drops. Rate needs to be slightly changed in
5112 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005113 double overshoot_factor = 2.0;
5114 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5115 // With bitrate adjuster, when need to overshoot even more to trigger
5116 // frame dropping.
5117 overshoot_factor *= 2;
5118 }
5119 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005120 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005121 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5122 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5123 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005124 num_dropped = 0;
5125 for (int i = 0; i < kNumFramesInRun; ++i) {
5126 video_source_.IncomingCapturedFrame(
5127 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5128 // Wait up to two frame durations for a frame to arrive.
5129 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5130 ++num_dropped;
5131 }
5132 timestamp_ms += 1000 / kFps;
5133 }
5134
Henrik Boström381d1092020-05-12 18:49:07 +02005135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005136 DataRate::BitsPerSec(kTargetBitrateBps),
5137 DataRate::BitsPerSec(kTargetBitrateBps),
5138 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005139
5140 // Target framerate should be still be near the expected target, despite
5141 // the frame drops.
5142 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5143
5144 // Frame drops should be within 5% of expected 50%.
5145 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005146
5147 video_stream_encoder_->Stop();
5148}
5149
5150TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5151 const int kFrameWidth = 320;
5152 const int kFrameHeight = 240;
5153 const int kActualInputFps = 24;
5154 const int kTargetBitrateBps = 120000;
5155
5156 ASSERT_GT(max_framerate_, kActualInputFps);
5157
5158 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5159 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005161 DataRate::BitsPerSec(kTargetBitrateBps),
5162 DataRate::BitsPerSec(kTargetBitrateBps),
5163 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005164
5165 // Insert 3 seconds of video, with an input fps lower than configured max.
5166 for (int i = 0; i < kActualInputFps * 3; ++i) {
5167 video_source_.IncomingCapturedFrame(
5168 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5169 // Wait up to two frame durations for a frame to arrive.
5170 WaitForEncodedFrame(timestamp_ms);
5171 timestamp_ms += 1000 / kActualInputFps;
5172 }
5173
5174 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5175
5176 video_stream_encoder_->Stop();
5177}
5178
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005179TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5180 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005181 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005182 DataRate::BitsPerSec(kTargetBitrateBps),
5183 DataRate::BitsPerSec(kTargetBitrateBps),
5184 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005185
5186 fake_encoder_.BlockNextEncode();
5187 video_source_.IncomingCapturedFrame(
5188 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5189 WaitForEncodedFrame(1);
5190 // On the very first frame full update should be forced.
5191 rect = fake_encoder_.GetLastUpdateRect();
5192 EXPECT_EQ(rect.offset_x, 0);
5193 EXPECT_EQ(rect.offset_y, 0);
5194 EXPECT_EQ(rect.height, codec_height_);
5195 EXPECT_EQ(rect.width, codec_width_);
5196 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5197 // call to ContinueEncode.
5198 video_source_.IncomingCapturedFrame(
5199 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5200 ExpectDroppedFrame();
5201 video_source_.IncomingCapturedFrame(
5202 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5203 ExpectDroppedFrame();
5204 fake_encoder_.ContinueEncode();
5205 WaitForEncodedFrame(3);
5206 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5207 rect = fake_encoder_.GetLastUpdateRect();
5208 EXPECT_EQ(rect.offset_x, 1);
5209 EXPECT_EQ(rect.offset_y, 0);
5210 EXPECT_EQ(rect.width, 10);
5211 EXPECT_EQ(rect.height, 1);
5212
5213 video_source_.IncomingCapturedFrame(
5214 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5215 WaitForEncodedFrame(4);
5216 // Previous frame was encoded, so no accumulation should happen.
5217 rect = fake_encoder_.GetLastUpdateRect();
5218 EXPECT_EQ(rect.offset_x, 0);
5219 EXPECT_EQ(rect.offset_y, 0);
5220 EXPECT_EQ(rect.width, 1);
5221 EXPECT_EQ(rect.height, 1);
5222
5223 video_stream_encoder_->Stop();
5224}
5225
Erik Språngd7329ca2019-02-21 21:19:53 +01005226TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005227 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005228 DataRate::BitsPerSec(kTargetBitrateBps),
5229 DataRate::BitsPerSec(kTargetBitrateBps),
5230 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005231
5232 // First frame is always keyframe.
5233 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5234 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005235 EXPECT_THAT(
5236 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005237 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005238
5239 // Insert delta frame.
5240 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5241 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005242 EXPECT_THAT(
5243 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005244 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005245
5246 // Request next frame be a key-frame.
5247 video_stream_encoder_->SendKeyFrame();
5248 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5249 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005250 EXPECT_THAT(
5251 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005252 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005253
5254 video_stream_encoder_->Stop();
5255}
5256
5257TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5258 // Setup simulcast with three streams.
5259 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005261 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5262 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5263 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005264 // Wait for all three layers before triggering event.
5265 sink_.SetNumExpectedLayers(3);
5266
5267 // First frame is always keyframe.
5268 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5269 WaitForEncodedFrame(1);
5270 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005271 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5272 VideoFrameType::kVideoFrameKey,
5273 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005274
5275 // Insert delta frame.
5276 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5277 WaitForEncodedFrame(2);
5278 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005279 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5280 VideoFrameType::kVideoFrameDelta,
5281 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005282
5283 // Request next frame be a key-frame.
5284 // Only first stream is configured to produce key-frame.
5285 video_stream_encoder_->SendKeyFrame();
5286 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5287 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005288
5289 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5290 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005291 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005292 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005293 VideoFrameType::kVideoFrameKey,
5294 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005295
5296 video_stream_encoder_->Stop();
5297}
5298
5299TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5300 // Configure internal source factory and setup test again.
5301 encoder_factory_.SetHasInternalSource(true);
5302 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005304 DataRate::BitsPerSec(kTargetBitrateBps),
5305 DataRate::BitsPerSec(kTargetBitrateBps),
5306 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005307
5308 // Call encoder directly, simulating internal source where encoded frame
5309 // callback in VideoStreamEncoder is called despite no OnFrame().
5310 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5311 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005312 EXPECT_THAT(
5313 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005314 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005315
Niels Möller8f7ce222019-03-21 15:43:58 +01005316 const std::vector<VideoFrameType> kDeltaFrame = {
5317 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005318 // Need to set timestamp manually since manually for injected frame.
5319 VideoFrame frame = CreateFrame(101, nullptr);
5320 frame.set_timestamp(101);
5321 fake_encoder_.InjectFrame(frame, false);
5322 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005323 EXPECT_THAT(
5324 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005325 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005326
5327 // Request key-frame. The forces a dummy frame down into the encoder.
5328 fake_encoder_.ExpectNullFrame();
5329 video_stream_encoder_->SendKeyFrame();
5330 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005331 EXPECT_THAT(
5332 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005333 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005334
5335 video_stream_encoder_->Stop();
5336}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005337
5338TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5339 // Configure internal source factory and setup test again.
5340 encoder_factory_.SetHasInternalSource(true);
5341 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005343 DataRate::BitsPerSec(kTargetBitrateBps),
5344 DataRate::BitsPerSec(kTargetBitrateBps),
5345 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005346
5347 int64_t timestamp = 1;
5348 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005349 image.SetEncodedData(
5350 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005351 image.capture_time_ms_ = ++timestamp;
5352 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5353 const int64_t kEncodeFinishDelayMs = 10;
5354 image.timing_.encode_start_ms = timestamp;
5355 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5356 fake_encoder_.InjectEncodedImage(image);
5357 // Wait for frame without incrementing clock.
5358 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5359 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5360 // capture timestamp should be kEncodeFinishDelayMs in the past.
5361 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5362 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5363 kEncodeFinishDelayMs);
5364
5365 video_stream_encoder_->Stop();
5366}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005367
5368TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5369 // Configure internal source factory and setup test again.
5370 encoder_factory_.SetHasInternalSource(true);
5371 ResetEncoder("H264", 1, 1, 1, false);
5372
5373 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5374 image._frameType = VideoFrameType::kVideoFrameKey;
5375
5376 CodecSpecificInfo codec_specific_info;
5377 codec_specific_info.codecType = kVideoCodecH264;
5378
5379 RTPFragmentationHeader fragmentation;
5380 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5381 fragmentation.fragmentationOffset[0] = 4;
5382 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5383
5384 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5385 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5386
5387 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5388 testing::ElementsAreArray(optimal_sps));
5389 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5390 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5391 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5392 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5393
5394 video_stream_encoder_->Stop();
5395}
5396
5397TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5398 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5399 0x00, 0x00, 0x03, 0x03, 0xF4,
5400 0x05, 0x03, 0xC7, 0xC0};
5401
5402 // Configure internal source factory and setup test again.
5403 encoder_factory_.SetHasInternalSource(true);
5404 ResetEncoder("H264", 1, 1, 1, false);
5405
5406 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5407 image._frameType = VideoFrameType::kVideoFrameKey;
5408
5409 CodecSpecificInfo codec_specific_info;
5410 codec_specific_info.codecType = kVideoCodecH264;
5411
5412 RTPFragmentationHeader fragmentation;
5413 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5414 fragmentation.fragmentationOffset[0] = 4;
5415 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5416
5417 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5418 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5419
5420 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5421 testing::ElementsAreArray(optimal_sps));
5422 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5423 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5424 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5425 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5426
5427 video_stream_encoder_->Stop();
5428}
5429
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005430TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5431 const int kFrameWidth = 1280;
5432 const int kFrameHeight = 720;
5433 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5434
Henrik Boström381d1092020-05-12 18:49:07 +02005435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005436 DataRate::BitsPerSec(kTargetBitrateBps),
5437 DataRate::BitsPerSec(kTargetBitrateBps),
5438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005439 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5440
5441 // Insert a first video frame. It should be dropped because of downscale in
5442 // resolution.
5443 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5444 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5445 frame.set_rotation(kVideoRotation_270);
5446 video_source_.IncomingCapturedFrame(frame);
5447
5448 ExpectDroppedFrame();
5449
5450 // Second frame is downscaled.
5451 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5452 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5453 frame.set_rotation(kVideoRotation_90);
5454 video_source_.IncomingCapturedFrame(frame);
5455
5456 WaitForEncodedFrame(timestamp_ms);
5457 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5458
5459 // Insert another frame, also downscaled.
5460 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5461 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5462 frame.set_rotation(kVideoRotation_180);
5463 video_source_.IncomingCapturedFrame(frame);
5464
5465 WaitForEncodedFrame(timestamp_ms);
5466 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5467
5468 video_stream_encoder_->Stop();
5469}
5470
Erik Språng5056af02019-09-02 15:53:11 +02005471TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5472 const int kFrameWidth = 320;
5473 const int kFrameHeight = 180;
5474
5475 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005477 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5478 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5479 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005480 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005481 /*rtt_ms=*/0,
5482 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005483
5484 // Insert a first video frame so that encoder gets configured.
5485 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5486 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5487 frame.set_rotation(kVideoRotation_270);
5488 video_source_.IncomingCapturedFrame(frame);
5489 WaitForEncodedFrame(timestamp_ms);
5490
5491 // Set a target rate below the minimum allowed by the codec settings.
5492 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005493 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5494 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005496 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005497 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005498 /*link_allocation=*/target_rate,
5499 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005500 /*rtt_ms=*/0,
5501 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005502 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5503
5504 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5505 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5506 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005507 DataRate allocation_sum =
5508 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005509 EXPECT_EQ(min_rate, allocation_sum);
5510 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5511
5512 video_stream_encoder_->Stop();
5513}
5514
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005515TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005517 DataRate::BitsPerSec(kTargetBitrateBps),
5518 DataRate::BitsPerSec(kTargetBitrateBps),
5519 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005520 // Capture a frame and wait for it to synchronize with the encoder thread.
5521 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5522 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5523 WaitForEncodedFrame(1);
5524
5525 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5526 ASSERT_TRUE(prev_rate_settings.has_value());
5527 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5528 kDefaultFramerate);
5529
5530 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5531 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5532 timestamp_ms += 1000 / kDefaultFramerate;
5533 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5534 WaitForEncodedFrame(timestamp_ms);
5535 }
5536 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5537 kDefaultFramerate);
5538 // Capture larger frame to trigger a reconfigure.
5539 codec_height_ *= 2;
5540 codec_width_ *= 2;
5541 timestamp_ms += 1000 / kDefaultFramerate;
5542 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5543 WaitForEncodedFrame(timestamp_ms);
5544
5545 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5546 auto current_rate_settings =
5547 fake_encoder_.GetAndResetLastRateControlSettings();
5548 // Ensure we have actually reconfigured twice
5549 // The rate settings should have been set again even though
5550 // they haven't changed.
5551 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005552 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005553
5554 video_stream_encoder_->Stop();
5555}
5556
philipeld9cc8c02019-09-16 14:53:40 +02005557struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005558 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5559 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5560 MOCK_METHOD(void,
5561 RequestEncoderSwitch,
5562 (const webrtc::SdpVideoFormat& format),
5563 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005564};
5565
5566TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5567 constexpr int kDontCare = 100;
5568
5569 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5570 video_send_config_.encoder_settings.encoder_switch_request_callback =
5571 &switch_callback;
5572 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5573 encoder_config.codec_type = kVideoCodecVP8;
5574 webrtc::test::ScopedFieldTrials field_trial(
5575 "WebRTC-NetworkCondition-EncoderSwitch/"
5576 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5577 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5578
5579 // Reset encoder for new configuration to take effect.
5580 ConfigureEncoder(std::move(encoder_config));
5581
5582 // Send one frame to trigger ReconfigureEncoder.
5583 video_source_.IncomingCapturedFrame(
5584 CreateFrame(kDontCare, kDontCare, kDontCare));
5585
5586 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005587 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5588 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005589 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005590 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005591
Henrik Boström381d1092020-05-12 18:49:07 +02005592 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005593 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5594 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5595 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005596 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005597 /*rtt_ms=*/0,
5598 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005599
5600 video_stream_encoder_->Stop();
5601}
5602
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005603TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5604 constexpr int kDontCare = 100;
5605
5606 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5607 video_send_config_.encoder_settings.encoder_switch_request_callback =
5608 &switch_callback;
5609 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5610 encoder_config.codec_type = kVideoCodecVP8;
5611 webrtc::test::ScopedFieldTrials field_trial(
5612 "WebRTC-NetworkCondition-EncoderSwitch/"
5613 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5614 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5615
5616 // Reset encoder for new configuration to take effect.
5617 ConfigureEncoder(std::move(encoder_config));
5618
5619 // Send one frame to trigger ReconfigureEncoder.
5620 video_source_.IncomingCapturedFrame(
5621 CreateFrame(kDontCare, kDontCare, kDontCare));
5622
5623 using Config = EncoderSwitchRequestCallback::Config;
5624 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5625 .Times(0);
5626
Henrik Boström381d1092020-05-12 18:49:07 +02005627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005628 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5629 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5630 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5631 /*fraction_lost=*/0,
5632 /*rtt_ms=*/0,
5633 /*cwnd_reduce_ratio=*/0);
5634
5635 video_stream_encoder_->Stop();
5636}
5637
philipeld9cc8c02019-09-16 14:53:40 +02005638TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5639 constexpr int kSufficientBitrateToNotDrop = 1000;
5640 constexpr int kHighRes = 500;
5641 constexpr int kLowRes = 100;
5642
5643 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5644 video_send_config_.encoder_settings.encoder_switch_request_callback =
5645 &switch_callback;
5646 webrtc::test::ScopedFieldTrials field_trial(
5647 "WebRTC-NetworkCondition-EncoderSwitch/"
5648 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5649 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5650 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5651 encoder_config.codec_type = kVideoCodecH264;
5652
5653 // Reset encoder for new configuration to take effect.
5654 ConfigureEncoder(std::move(encoder_config));
5655
5656 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5657 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5658 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005660 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5661 /*stable_target_bitrate=*/
5662 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5663 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005664 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005665 /*rtt_ms=*/0,
5666 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005667
5668 // Send one frame to trigger ReconfigureEncoder.
5669 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5670 WaitForEncodedFrame(1);
5671
5672 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005673 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5674 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005675 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005676 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005677
5678 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5679 WaitForEncodedFrame(2);
5680
5681 video_stream_encoder_->Stop();
5682}
5683
philipel9b058032020-02-10 11:30:00 +01005684TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5685 constexpr int kDontCare = 100;
5686 StrictMock<MockEncoderSelector> encoder_selector;
5687 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5688 &fake_encoder_, &encoder_selector);
5689 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5690
5691 // Reset encoder for new configuration to take effect.
5692 ConfigureEncoder(video_encoder_config_.Copy());
5693
5694 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5695
5696 video_source_.IncomingCapturedFrame(
5697 CreateFrame(kDontCare, kDontCare, kDontCare));
5698 video_stream_encoder_->Stop();
5699
5700 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5701 // to it's factory, so in order for the encoder instance in the
5702 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5703 // reset the |video_stream_encoder_| here.
5704 video_stream_encoder_.reset();
5705}
5706
5707TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5708 constexpr int kDontCare = 100;
5709
5710 NiceMock<MockEncoderSelector> encoder_selector;
5711 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5712 video_send_config_.encoder_settings.encoder_switch_request_callback =
5713 &switch_callback;
5714 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5715 &fake_encoder_, &encoder_selector);
5716 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5717
5718 // Reset encoder for new configuration to take effect.
5719 ConfigureEncoder(video_encoder_config_.Copy());
5720
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005721 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005722 .WillByDefault(Return(SdpVideoFormat("AV1")));
5723 EXPECT_CALL(switch_callback,
5724 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5725 Field(&SdpVideoFormat::name, "AV1"))));
5726
Henrik Boström381d1092020-05-12 18:49:07 +02005727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005728 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5729 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5730 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005731 /*fraction_lost=*/0,
5732 /*rtt_ms=*/0,
5733 /*cwnd_reduce_ratio=*/0);
5734
5735 video_stream_encoder_->Stop();
5736}
5737
5738TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5739 constexpr int kSufficientBitrateToNotDrop = 1000;
5740 constexpr int kDontCare = 100;
5741
5742 NiceMock<MockVideoEncoder> video_encoder;
5743 NiceMock<MockEncoderSelector> encoder_selector;
5744 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5745 video_send_config_.encoder_settings.encoder_switch_request_callback =
5746 &switch_callback;
5747 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5748 &video_encoder, &encoder_selector);
5749 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5750
5751 // Reset encoder for new configuration to take effect.
5752 ConfigureEncoder(video_encoder_config_.Copy());
5753
5754 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5755 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5756 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005757 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005758 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5759 /*stable_target_bitrate=*/
5760 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5761 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005762 /*fraction_lost=*/0,
5763 /*rtt_ms=*/0,
5764 /*cwnd_reduce_ratio=*/0);
5765
5766 ON_CALL(video_encoder, Encode(_, _))
5767 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5768 ON_CALL(encoder_selector, OnEncoderBroken())
5769 .WillByDefault(Return(SdpVideoFormat("AV2")));
5770
5771 rtc::Event encode_attempted;
5772 EXPECT_CALL(switch_callback,
5773 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5774 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5775 EXPECT_EQ(format.name, "AV2");
5776 encode_attempted.Set();
5777 });
5778
5779 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5780 encode_attempted.Wait(3000);
5781
5782 video_stream_encoder_->Stop();
5783
5784 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5785 // to it's factory, so in order for the encoder instance in the
5786 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5787 // reset the |video_stream_encoder_| here.
5788 video_stream_encoder_.reset();
5789}
5790
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005791TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005792 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005793 const int kFrameWidth = 320;
5794 const int kFrameHeight = 180;
5795
5796 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005797 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005798 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005799 /*target_bitrate=*/rate,
5800 /*stable_target_bitrate=*/rate,
5801 /*link_allocation=*/rate,
5802 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005803 /*rtt_ms=*/0,
5804 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005805
5806 // Insert a first video frame so that encoder gets configured.
5807 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5808 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5809 frame.set_rotation(kVideoRotation_270);
5810 video_source_.IncomingCapturedFrame(frame);
5811 WaitForEncodedFrame(timestamp_ms);
5812 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5813
5814 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005815 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005817 /*target_bitrate=*/new_stable_rate,
5818 /*stable_target_bitrate=*/new_stable_rate,
5819 /*link_allocation=*/rate,
5820 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005821 /*rtt_ms=*/0,
5822 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005823 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5824 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5825 video_stream_encoder_->Stop();
5826}
5827
5828TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005829 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005830 const int kFrameWidth = 320;
5831 const int kFrameHeight = 180;
5832
5833 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005834 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005836 /*target_bitrate=*/rate,
5837 /*stable_target_bitrate=*/rate,
5838 /*link_allocation=*/rate,
5839 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005840 /*rtt_ms=*/0,
5841 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005842
5843 // Insert a first video frame so that encoder gets configured.
5844 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5845 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5846 frame.set_rotation(kVideoRotation_270);
5847 video_source_.IncomingCapturedFrame(frame);
5848 WaitForEncodedFrame(timestamp_ms);
5849 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5850
5851 // Set a higher target rate without changing the link_allocation. Should not
5852 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005853 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005855 /*target_bitrate=*/rate,
5856 /*stable_target_bitrate=*/new_stable_rate,
5857 /*link_allocation=*/rate,
5858 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005859 /*rtt_ms=*/0,
5860 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005861 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5862 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5863 video_stream_encoder_->Stop();
5864}
5865
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005866TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5867 test::ScopedFieldTrials field_trials(
5868 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5869 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5870 const int kFramerateFps = 30;
5871 const int kWidth = 1920;
5872 const int kHeight = 1080;
5873 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5874 // Works on screenshare mode.
5875 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5876 // We rely on the automatic resolution adaptation, but we handle framerate
5877 // adaptation manually by mocking the stats proxy.
5878 video_source_.set_adaptation_enabled(true);
5879
5880 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005881 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005882 DataRate::BitsPerSec(kTargetBitrateBps),
5883 DataRate::BitsPerSec(kTargetBitrateBps),
5884 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005885 video_stream_encoder_->SetSource(&video_source_,
5886 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005887 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005888
5889 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5890 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5891
5892 // Pass enough frames with the full update to trigger animation detection.
5893 for (int i = 0; i < kNumFrames; ++i) {
5894 int64_t timestamp_ms =
5895 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5896 frame.set_ntp_time_ms(timestamp_ms);
5897 frame.set_timestamp_us(timestamp_ms * 1000);
5898 video_source_.IncomingCapturedFrame(frame);
5899 WaitForEncodedFrame(timestamp_ms);
5900 }
5901
5902 // Resolution should be limited.
5903 rtc::VideoSinkWants expected;
5904 expected.max_framerate_fps = kFramerateFps;
5905 expected.max_pixel_count = 1280 * 720 + 1;
5906 VerifyFpsEqResolutionLt(video_source_.sink_wants(), expected);
5907
5908 // Pass one frame with no known update.
5909 // Resolution cap should be removed immediately.
5910 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5911 frame.set_ntp_time_ms(timestamp_ms);
5912 frame.set_timestamp_us(timestamp_ms * 1000);
5913 frame.clear_update_rect();
5914
5915 video_source_.IncomingCapturedFrame(frame);
5916 WaitForEncodedFrame(timestamp_ms);
5917
5918 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005919 EXPECT_THAT(video_source_.sink_wants(),
5920 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005921
5922 video_stream_encoder_->Stop();
5923}
5924
perkj26091b12016-09-01 01:17:40 -07005925} // namespace webrtc