blob: 4ff7ddcc8f18e5cf07661ad69518a6e6f87fae4b [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010029#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020030#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070031#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080032#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020033#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020034#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010035#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080036#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020037#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020038#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080039#include "rtc_base/ref_counted_object.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010040#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020041#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020042#include "system_wrappers/include/sleep.h"
43#include "test/encoder_settings.h"
44#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020045#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010046#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020047#include "test/gmock.h"
48#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020049#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070051
52namespace webrtc {
53
sprang57c2fff2017-01-16 06:24:02 -080054using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020055using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020056using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020057using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020058using ::testing::Ge;
59using ::testing::Gt;
60using ::testing::Le;
61using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010062using ::testing::Matcher;
63using ::testing::NiceMock;
64using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020065using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080066
perkj803d97f2016-11-01 11:45:46 -070067namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020068const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010069const int kQpLow = 1;
70const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020071const int kMinFramerateFps = 2;
72const int kMinBalancedFramerateFps = 7;
73const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080074const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010075const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020076const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010077const uint32_t kSimulcastTargetBitrateBps = 3150000;
78const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080079const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070080const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020081const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020082const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020083const VideoEncoder::ResolutionBitrateLimits
84 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
85const VideoEncoder::ResolutionBitrateLimits
86 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080087
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020088uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
89 0x00, 0x00, 0x03, 0x03, 0xF4,
90 0x05, 0x03, 0xC7, 0xE0, 0x1B,
91 0x41, 0x10, 0x8D, 0x00};
92
perkj803d97f2016-11-01 11:45:46 -070093class TestBuffer : public webrtc::I420Buffer {
94 public:
95 TestBuffer(rtc::Event* event, int width, int height)
96 : I420Buffer(width, height), event_(event) {}
97
98 private:
99 friend class rtc::RefCountedObject<TestBuffer>;
100 ~TestBuffer() override {
101 if (event_)
102 event_->Set();
103 }
104 rtc::Event* const event_;
105};
106
Noah Richards51db4212019-06-12 06:59:12 -0700107// A fake native buffer that can't be converted to I420.
108class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
109 public:
110 FakeNativeBuffer(rtc::Event* event, int width, int height)
111 : event_(event), width_(width), height_(height) {}
112 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
113 int width() const override { return width_; }
114 int height() const override { return height_; }
115 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
116 return nullptr;
117 }
118
119 private:
120 friend class rtc::RefCountedObject<FakeNativeBuffer>;
121 ~FakeNativeBuffer() override {
122 if (event_)
123 event_->Set();
124 }
125 rtc::Event* const event_;
126 const int width_;
127 const int height_;
128};
129
Niels Möller7dc26b72017-12-06 10:27:48 +0100130class CpuOveruseDetectorProxy : public OveruseFrameDetector {
131 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200132 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
133 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200134 last_target_framerate_fps_(-1),
135 framerate_updated_event_(true /* manual_reset */,
136 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100137 virtual ~CpuOveruseDetectorProxy() {}
138
139 void OnTargetFramerateUpdated(int framerate_fps) override {
140 rtc::CritScope cs(&lock_);
141 last_target_framerate_fps_ = framerate_fps;
142 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200143 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 }
145
146 int GetLastTargetFramerate() {
147 rtc::CritScope cs(&lock_);
148 return last_target_framerate_fps_;
149 }
150
Niels Möller4db138e2018-04-19 09:04:13 +0200151 CpuOveruseOptions GetOptions() { return options_; }
152
Henrik Boström381d1092020-05-12 18:49:07 +0200153 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
154
Niels Möller7dc26b72017-12-06 10:27:48 +0100155 private:
156 rtc::CriticalSection lock_;
157 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200158 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100159};
160
Henrik Boström91aa7322020-04-28 12:24:33 +0200161class FakeQualityScalerQpUsageHandlerCallback
162 : public QualityScalerQpUsageHandlerCallbackInterface {
163 public:
164 FakeQualityScalerQpUsageHandlerCallback()
Henrik Boström5bf60e42020-05-13 15:20:25 +0200165 : QualityScalerQpUsageHandlerCallbackInterface(),
166 qp_usage_handled_event_(/*manual_reset=*/true,
167 /*initially_signaled=*/false),
168 clear_qp_samples_result_(absl::nullopt) {}
169 ~FakeQualityScalerQpUsageHandlerCallback() override {
170 RTC_DCHECK(clear_qp_samples_result_.has_value());
171 }
Henrik Boström91aa7322020-04-28 12:24:33 +0200172
173 void OnQpUsageHandled(bool clear_qp_samples) override {
174 clear_qp_samples_result_ = clear_qp_samples;
Henrik Boström5bf60e42020-05-13 15:20:25 +0200175 qp_usage_handled_event_.Set();
Henrik Boström91aa7322020-04-28 12:24:33 +0200176 }
177
Henrik Boström5bf60e42020-05-13 15:20:25 +0200178 bool WaitForQpUsageHandled() { return qp_usage_handled_event_.Wait(5000); }
179
Henrik Boström91aa7322020-04-28 12:24:33 +0200180 absl::optional<bool> clear_qp_samples_result() const {
181 return clear_qp_samples_result_;
182 }
183
184 private:
Henrik Boström5bf60e42020-05-13 15:20:25 +0200185 rtc::Event qp_usage_handled_event_;
Henrik Boström91aa7322020-04-28 12:24:33 +0200186 absl::optional<bool> clear_qp_samples_result_;
187};
188
Henrik Boström381d1092020-05-12 18:49:07 +0200189class VideoSourceRestrictionsUpdatedListener
190 : public ResourceAdaptationProcessorListener {
191 public:
192 VideoSourceRestrictionsUpdatedListener()
193 : was_restrictions_updated_(false), restrictions_updated_event_() {}
194 ~VideoSourceRestrictionsUpdatedListener() override {
195 RTC_DCHECK(was_restrictions_updated_);
196 }
197
198 rtc::Event* restrictions_updated_event() {
199 return &restrictions_updated_event_;
200 }
201
202 // ResourceAdaptationProcessorListener implementation.
203 void OnVideoSourceRestrictionsUpdated(
204 VideoSourceRestrictions restrictions,
205 const VideoAdaptationCounters& adaptation_counters,
206 rtc::scoped_refptr<Resource> reason) override {
207 was_restrictions_updated_ = true;
208 restrictions_updated_event_.Set();
209 }
210
211 private:
212 bool was_restrictions_updated_;
213 rtc::Event restrictions_updated_event_;
214};
215
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200216auto WantsFps(Matcher<int> fps_matcher) {
217 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
218 fps_matcher);
219}
220
221auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
222 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
223 AllOf(max_pixel_matcher, Gt(0)));
224}
225
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200226auto ResolutionMax() {
227 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200228 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200229 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
230 Eq(absl::nullopt)));
231}
232
233auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200234 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200235}
236
237auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239}
240
241auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200242 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200243}
244
245auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200246 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200247}
248
249auto FpsMaxResolutionMax() {
250 return AllOf(FpsMax(), ResolutionMax());
251}
252
253auto UnlimitedSinkWants() {
254 return AllOf(FpsUnlimited(), ResolutionMax());
255}
256
257auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
258 Matcher<int> fps_range_matcher;
259
260 if (last_frame_pixels <= 320 * 240) {
261 fps_range_matcher = AllOf(Ge(7), Le(10));
262 } else if (last_frame_pixels <= 480 * 270) {
263 fps_range_matcher = AllOf(Ge(10), Le(15));
264 } else if (last_frame_pixels <= 640 * 480) {
265 fps_range_matcher = Ge(15);
266 } else {
267 fps_range_matcher = Eq(kDefaultFramerate);
268 }
269 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
270 fps_range_matcher);
271}
272
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200273auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
274 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
275 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
276}
277
278auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
279 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
280}
281
282auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
283 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
284}
285
286auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
287 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
288 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
289}
290
291auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
292 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
293 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
294}
295
296auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
299}
300
301auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
304}
305
mflodmancc3d4422017-08-03 08:27:51 -0700306class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700307 public:
Niels Möller213618e2018-07-24 09:29:58 +0200308 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200309 const VideoStreamEncoderSettings& settings,
310 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100311 : VideoStreamEncoder(Clock::GetRealTimeClock(),
312 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200313 stats_proxy,
314 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200315 std::unique_ptr<OveruseFrameDetector>(
316 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100317 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100318 task_queue_factory),
Henrik Boströmc55516d2020-05-11 16:29:22 +0200319 fake_cpu_resource_(new FakeResource("FakeResource[CPU]")),
320 fake_quality_resource_(new FakeResource("FakeResource[QP]")) {
Henrik Boström381d1092020-05-12 18:49:07 +0200321 fake_cpu_resource_->Initialize(encoder_queue(),
322 resource_adaptation_queue());
323 fake_quality_resource_->Initialize(encoder_queue(),
324 resource_adaptation_queue());
Henrik Boströmc55516d2020-05-11 16:29:22 +0200325 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200326 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200327 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100328 }
perkj803d97f2016-11-01 11:45:46 -0700329
Henrik Boström381d1092020-05-12 18:49:07 +0200330 void SetSourceAndWaitForRestrictionsUpdated(
331 rtc::VideoSourceInterface<VideoFrame>* source,
332 const DegradationPreference& degradation_preference) {
333 VideoSourceRestrictionsUpdatedListener listener;
334 AddAdaptationListenerForTesting(&listener);
335 SetSource(source, degradation_preference);
336 listener.restrictions_updated_event()->Wait(5000);
337 RemoveAdaptationListenerForTesting(&listener);
338 }
339
340 void SetSourceAndWaitForFramerateUpdated(
341 rtc::VideoSourceInterface<VideoFrame>* source,
342 const DegradationPreference& degradation_preference) {
343 overuse_detector_proxy_->framerate_updated_event()->Reset();
344 SetSource(source, degradation_preference);
345 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
346 }
347
348 void OnBitrateUpdatedAndWaitForManagedResources(
349 DataRate target_bitrate,
350 DataRate stable_target_bitrate,
351 DataRate link_allocation,
352 uint8_t fraction_lost,
353 int64_t round_trip_time_ms,
354 double cwnd_reduce_ratio) {
355 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
356 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
357 // Bitrate is updated on the encoder queue.
358 WaitUntilTaskQueueIsIdle();
359 // Give the managed resources time to react to the new bitrate.
360 // TODO(hbos): Can we await an appropriate event instead?
361 WaitUntilAdaptationTaskQueueIsIdle();
362 }
363
364 void WaitUntilAdaptationTaskQueueIsIdle() {
365 rtc::Event event;
366 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
367 ASSERT_TRUE(event.Wait(5000));
368 }
369
kthelgason2fc52542017-03-03 00:24:41 -0800370 // This is used as a synchronisation mechanism, to make sure that the
371 // encoder queue is not blocked before we start sending it frames.
372 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100373 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200374 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800375 ASSERT_TRUE(event.Wait(5000));
376 }
377
Henrik Boström91aa7322020-04-28 12:24:33 +0200378 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200379 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200380 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200381 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 fake_cpu_resource_->set_usage_state(ResourceUsageState::kOveruse);
383 event.Set();
384 });
385 ASSERT_TRUE(event.Wait(5000));
386 }
387 void TriggerCpuUnderuse() {
388 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200389 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200390 fake_cpu_resource_->set_usage_state(ResourceUsageState::kUnderuse);
391 event.Set();
392 });
393 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200394 }
kthelgason876222f2016-11-29 01:44:11 -0800395
Henrik Boström91aa7322020-04-28 12:24:33 +0200396 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200397 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200398 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200399 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200400 fake_quality_resource_->set_usage_state(ResourceUsageState::kOveruse);
401 event.Set();
402 });
403 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200404 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200405 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200406 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200407 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 fake_quality_resource_->set_usage_state(ResourceUsageState::kUnderuse);
409 event.Set();
410 });
411 ASSERT_TRUE(event.Wait(5000));
412 }
413
414 // Fakes high QP resource usage measurements on the real
415 // QualityScalerResource. Returns whether or not QP samples would have been
416 // cleared if this had been a real signal from the QualityScaler.
417 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
419 new FakeQualityScalerQpUsageHandlerCallback();
Henrik Boström5bf60e42020-05-13 15:20:25 +0200420 encoder_queue()->PostTask([this, callback] {
421 // This will cause a "ping" between adaptation task queue and encoder
422 // queue. When we have the result, the |callback| will be notified.
Henrik Boström91aa7322020-04-28 12:24:33 +0200423 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
Henrik Boström91aa7322020-04-28 12:24:33 +0200424 });
Henrik Boström5bf60e42020-05-13 15:20:25 +0200425 EXPECT_TRUE(callback->WaitForQpUsageHandled());
Henrik Boström91aa7322020-04-28 12:24:33 +0200426 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
427 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200428 }
sprangfda496a2017-06-15 04:21:07 -0700429
Niels Möller7dc26b72017-12-06 10:27:48 +0100430 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200431 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
432 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
perkj803d97f2016-11-01 11:45:46 -0700433};
434
asapersson5f7226f2016-11-25 04:37:00 -0800435class VideoStreamFactory
436 : public VideoEncoderConfig::VideoStreamFactoryInterface {
437 public:
sprangfda496a2017-06-15 04:21:07 -0700438 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
439 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800440 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700441 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800442 }
443
444 private:
445 std::vector<VideoStream> CreateEncoderStreams(
446 int width,
447 int height,
448 const VideoEncoderConfig& encoder_config) override {
449 std::vector<VideoStream> streams =
450 test::CreateVideoStreams(width, height, encoder_config);
451 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100452 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700453 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800454 }
455 return streams;
456 }
sprangfda496a2017-06-15 04:21:07 -0700457
asapersson5f7226f2016-11-25 04:37:00 -0800458 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700459 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800460};
461
Noah Richards51db4212019-06-12 06:59:12 -0700462// Simulates simulcast behavior and makes highest stream resolutions divisible
463// by 4.
464class CroppingVideoStreamFactory
465 : public VideoEncoderConfig::VideoStreamFactoryInterface {
466 public:
467 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
468 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
469 EXPECT_GT(num_temporal_layers, 0u);
470 EXPECT_GT(framerate, 0);
471 }
472
473 private:
474 std::vector<VideoStream> CreateEncoderStreams(
475 int width,
476 int height,
477 const VideoEncoderConfig& encoder_config) override {
478 std::vector<VideoStream> streams = test::CreateVideoStreams(
479 width - width % 4, height - height % 4, encoder_config);
480 for (VideoStream& stream : streams) {
481 stream.num_temporal_layers = num_temporal_layers_;
482 stream.max_framerate = framerate_;
483 }
484 return streams;
485 }
486
487 const size_t num_temporal_layers_;
488 const int framerate_;
489};
490
sprangb1ca0732017-02-01 08:38:12 -0800491class AdaptingFrameForwarder : public test::FrameForwarder {
492 public:
493 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700494 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800495
496 void set_adaptation_enabled(bool enabled) {
497 rtc::CritScope cs(&crit_);
498 adaptation_enabled_ = enabled;
499 }
500
asaperssonfab67072017-04-04 05:51:49 -0700501 bool adaption_enabled() const {
sprangb1ca0732017-02-01 08:38:12 -0800502 rtc::CritScope cs(&crit_);
503 return adaptation_enabled_;
504 }
505
asapersson09f05612017-05-15 23:40:18 -0700506 rtc::VideoSinkWants last_wants() const {
507 rtc::CritScope cs(&crit_);
508 return last_wants_;
509 }
510
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200511 absl::optional<int> last_sent_width() const { return last_width_; }
512 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800513
sprangb1ca0732017-02-01 08:38:12 -0800514 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
515 int cropped_width = 0;
516 int cropped_height = 0;
517 int out_width = 0;
518 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700519 if (adaption_enabled()) {
520 if (adapter_.AdaptFrameResolution(
521 video_frame.width(), video_frame.height(),
522 video_frame.timestamp_us() * 1000, &cropped_width,
523 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100524 VideoFrame adapted_frame =
525 VideoFrame::Builder()
526 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
527 nullptr, out_width, out_height))
528 .set_timestamp_rtp(99)
529 .set_timestamp_ms(99)
530 .set_rotation(kVideoRotation_0)
531 .build();
sprangc5d62e22017-04-02 23:53:04 -0700532 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100533 if (video_frame.has_update_rect()) {
534 adapted_frame.set_update_rect(
535 video_frame.update_rect().ScaleWithFrame(
536 video_frame.width(), video_frame.height(), 0, 0,
537 video_frame.width(), video_frame.height(), out_width,
538 out_height));
539 }
sprangc5d62e22017-04-02 23:53:04 -0700540 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800541 last_width_.emplace(adapted_frame.width());
542 last_height_.emplace(adapted_frame.height());
543 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200544 last_width_ = absl::nullopt;
545 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700546 }
sprangb1ca0732017-02-01 08:38:12 -0800547 } else {
548 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800549 last_width_.emplace(video_frame.width());
550 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800551 }
552 }
553
554 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
555 const rtc::VideoSinkWants& wants) override {
556 rtc::CritScope cs(&crit_);
asapersson09f05612017-05-15 23:40:18 -0700557 last_wants_ = sink_wants();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100558 adapter_.OnSinkWants(wants);
sprangb1ca0732017-02-01 08:38:12 -0800559 test::FrameForwarder::AddOrUpdateSink(sink, wants);
560 }
sprangb1ca0732017-02-01 08:38:12 -0800561 cricket::VideoAdapter adapter_;
danilchapa37de392017-09-09 04:17:22 -0700562 bool adaptation_enabled_ RTC_GUARDED_BY(crit_);
563 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(crit_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200564 absl::optional<int> last_width_;
565 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800566};
sprangc5d62e22017-04-02 23:53:04 -0700567
Niels Möller213618e2018-07-24 09:29:58 +0200568// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700569class MockableSendStatisticsProxy : public SendStatisticsProxy {
570 public:
571 MockableSendStatisticsProxy(Clock* clock,
572 const VideoSendStream::Config& config,
573 VideoEncoderConfig::ContentType content_type)
574 : SendStatisticsProxy(clock, config, content_type) {}
575
576 VideoSendStream::Stats GetStats() override {
577 rtc::CritScope cs(&lock_);
578 if (mock_stats_)
579 return *mock_stats_;
580 return SendStatisticsProxy::GetStats();
581 }
582
Niels Möller213618e2018-07-24 09:29:58 +0200583 int GetInputFrameRate() const override {
584 rtc::CritScope cs(&lock_);
585 if (mock_stats_)
586 return mock_stats_->input_frame_rate;
587 return SendStatisticsProxy::GetInputFrameRate();
588 }
sprangc5d62e22017-04-02 23:53:04 -0700589 void SetMockStats(const VideoSendStream::Stats& stats) {
590 rtc::CritScope cs(&lock_);
591 mock_stats_.emplace(stats);
592 }
593
594 void ResetMockStats() {
595 rtc::CritScope cs(&lock_);
596 mock_stats_.reset();
597 }
598
599 private:
600 rtc::CriticalSection lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200601 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700602};
603
sprang4847ae62017-06-27 07:06:52 -0700604class MockBitrateObserver : public VideoBitrateAllocationObserver {
605 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200606 MOCK_METHOD(void,
607 OnBitrateAllocationUpdated,
608 (const VideoBitrateAllocation&),
609 (override));
sprang4847ae62017-06-27 07:06:52 -0700610};
611
philipel9b058032020-02-10 11:30:00 +0100612class MockEncoderSelector
613 : public VideoEncoderFactory::EncoderSelectorInterface {
614 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200615 MOCK_METHOD(void,
616 OnCurrentEncoder,
617 (const SdpVideoFormat& format),
618 (override));
619 MOCK_METHOD(absl::optional<SdpVideoFormat>,
620 OnAvailableBitrate,
621 (const DataRate& rate),
622 (override));
623 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100624};
625
perkj803d97f2016-11-01 11:45:46 -0700626} // namespace
627
mflodmancc3d4422017-08-03 08:27:51 -0700628class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700629 public:
630 static const int kDefaultTimeoutMs = 30 * 1000;
631
mflodmancc3d4422017-08-03 08:27:51 -0700632 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700633 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700634 codec_width_(320),
635 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200636 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200637 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700638 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200639 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700640 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700641 Clock::GetRealTimeClock(),
642 video_send_config_,
643 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700644 sink_(&fake_encoder_) {}
645
646 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700647 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700648 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200649 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800650 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200651 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200652 video_send_config_.rtp.payload_name = "FAKE";
653 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700654
Per512ecb32016-09-23 15:52:06 +0200655 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200656 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700657 video_encoder_config.video_stream_factory =
658 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100659 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700660
661 // Framerate limit is specified by the VideoStreamFactory.
662 std::vector<VideoStream> streams =
663 video_encoder_config.video_stream_factory->CreateEncoderStreams(
664 codec_width_, codec_height_, video_encoder_config);
665 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100666 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700667
Niels Möllerf1338562018-04-26 09:51:47 +0200668 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800669 }
670
Niels Möllerf1338562018-04-26 09:51:47 +0200671 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700672 if (video_stream_encoder_)
673 video_stream_encoder_->Stop();
674 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200675 stats_proxy_.get(), video_send_config_.encoder_settings,
676 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700677 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
678 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700679 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700680 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
681 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200682 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800684 }
685
686 void ResetEncoder(const std::string& payload_name,
687 size_t num_streams,
688 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700689 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700690 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200691 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800692
693 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200694 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800695 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100696 video_encoder_config.max_bitrate_bps =
697 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800698 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700699 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
700 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700701 video_encoder_config.content_type =
702 screenshare ? VideoEncoderConfig::ContentType::kScreen
703 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700704 if (payload_name == "VP9") {
705 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
706 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
707 video_encoder_config.encoder_specific_settings =
708 new rtc::RefCountedObject<
709 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
710 }
Niels Möllerf1338562018-04-26 09:51:47 +0200711 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700712 }
713
sprang57c2fff2017-01-16 06:24:02 -0800714 VideoFrame CreateFrame(int64_t ntp_time_ms,
715 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100716 VideoFrame frame =
717 VideoFrame::Builder()
718 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
719 destruction_event, codec_width_, codec_height_))
720 .set_timestamp_rtp(99)
721 .set_timestamp_ms(99)
722 .set_rotation(kVideoRotation_0)
723 .build();
sprang57c2fff2017-01-16 06:24:02 -0800724 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700725 return frame;
726 }
727
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100728 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
729 rtc::Event* destruction_event,
730 int offset_x) const {
731 VideoFrame frame =
732 VideoFrame::Builder()
733 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
734 destruction_event, codec_width_, codec_height_))
735 .set_timestamp_rtp(99)
736 .set_timestamp_ms(99)
737 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100738 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100739 .build();
740 frame.set_ntp_time_ms(ntp_time_ms);
741 return frame;
742 }
743
sprang57c2fff2017-01-16 06:24:02 -0800744 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100745 VideoFrame frame =
746 VideoFrame::Builder()
747 .set_video_frame_buffer(
748 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
749 .set_timestamp_rtp(99)
750 .set_timestamp_ms(99)
751 .set_rotation(kVideoRotation_0)
752 .build();
sprang57c2fff2017-01-16 06:24:02 -0800753 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700754 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700755 return frame;
756 }
757
Noah Richards51db4212019-06-12 06:59:12 -0700758 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
759 rtc::Event* destruction_event,
760 int width,
761 int height) const {
762 VideoFrame frame =
763 VideoFrame::Builder()
764 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
765 destruction_event, width, height))
766 .set_timestamp_rtp(99)
767 .set_timestamp_ms(99)
768 .set_rotation(kVideoRotation_0)
769 .build();
770 frame.set_ntp_time_ms(ntp_time_ms);
771 return frame;
772 }
773
774 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
775 rtc::Event* destruction_event) const {
776 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
777 codec_height_);
778 }
779
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100780 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
781 MockBitrateObserver bitrate_observer;
782 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
783
784 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
785 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100787 DataRate::BitsPerSec(kTargetBitrateBps),
788 DataRate::BitsPerSec(kTargetBitrateBps),
789 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100790
791 video_source_.IncomingCapturedFrame(
792 CreateFrame(1, codec_width_, codec_height_));
793 WaitForEncodedFrame(1);
794 }
795
sprang4847ae62017-06-27 07:06:52 -0700796 void WaitForEncodedFrame(int64_t expected_ntp_time) {
797 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100798 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700799 }
800
801 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
802 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100803 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700804 return ok;
805 }
806
807 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
808 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100809 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700810 }
811
812 void ExpectDroppedFrame() {
813 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100814 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700815 }
816
817 bool WaitForFrame(int64_t timeout_ms) {
818 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100819 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700820 return ok;
821 }
822
perkj26091b12016-09-01 01:17:40 -0700823 class TestEncoder : public test::FakeEncoder {
824 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100825 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700826
asaperssonfab67072017-04-04 05:51:49 -0700827 VideoCodec codec_config() const {
brandtre78d2662017-01-16 05:57:16 -0800828 rtc::CritScope lock(&crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700829 return config_;
830 }
831
832 void BlockNextEncode() {
brandtre78d2662017-01-16 05:57:16 -0800833 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700834 block_next_encode_ = true;
835 }
836
Erik Språngaed30702018-11-05 12:57:17 +0100837 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
kthelgason2fc52542017-03-03 00:24:41 -0800838 rtc::CritScope lock(&local_crit_sect_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100839 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100840 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100841 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100842 info.scaling_settings = VideoEncoder::ScalingSettings(
843 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 }
845 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100846 for (int i = 0; i < kMaxSpatialLayers; ++i) {
847 if (temporal_layers_supported_[i]) {
848 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
849 info.fps_allocation[i].resize(num_layers);
850 }
851 }
Erik Språngaed30702018-11-05 12:57:17 +0100852 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200853
854 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100855 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100856 return info;
kthelgason876222f2016-11-29 01:44:11 -0800857 }
858
Erik Språngb7cb7b52019-02-26 15:52:33 +0100859 int32_t RegisterEncodeCompleteCallback(
860 EncodedImageCallback* callback) override {
861 rtc::CritScope lock(&local_crit_sect_);
862 encoded_image_callback_ = callback;
863 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
864 }
865
perkjfa10b552016-10-02 23:45:26 -0700866 void ContinueEncode() { continue_encode_event_.Set(); }
867
868 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
869 uint32_t timestamp) const {
brandtre78d2662017-01-16 05:57:16 -0800870 rtc::CritScope lock(&local_crit_sect_);
perkjfa10b552016-10-02 23:45:26 -0700871 EXPECT_EQ(timestamp_, timestamp);
872 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
873 }
874
kthelgason2fc52542017-03-03 00:24:41 -0800875 void SetQualityScaling(bool b) {
876 rtc::CritScope lock(&local_crit_sect_);
877 quality_scaling_ = b;
878 }
kthelgasonad9010c2017-02-14 00:46:51 -0800879
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100880 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
881 rtc::CritScope lock(&local_crit_sect_);
882 requested_resolution_alignment_ = requested_resolution_alignment;
883 }
884
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100885 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
886 rtc::CritScope lock(&local_crit_sect_);
887 is_hardware_accelerated_ = is_hardware_accelerated;
888 }
889
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100890 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
891 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
892 rtc::CritScope lock(&local_crit_sect_);
893 temporal_layers_supported_[spatial_idx] = supported;
894 }
895
Sergey Silkin6456e352019-07-08 17:56:40 +0200896 void SetResolutionBitrateLimits(
897 std::vector<ResolutionBitrateLimits> thresholds) {
898 rtc::CritScope cs(&local_crit_sect_);
899 resolution_bitrate_limits_ = thresholds;
900 }
901
sprangfe627f32017-03-29 08:24:59 -0700902 void ForceInitEncodeFailure(bool force_failure) {
903 rtc::CritScope lock(&local_crit_sect_);
904 force_init_encode_failed_ = force_failure;
905 }
906
Niels Möller6bb5ab92019-01-11 11:11:10 +0100907 void SimulateOvershoot(double rate_factor) {
908 rtc::CritScope lock(&local_crit_sect_);
909 rate_factor_ = rate_factor;
910 }
911
Erik Språngd7329ca2019-02-21 21:19:53 +0100912 uint32_t GetLastFramerate() const {
Niels Möller6bb5ab92019-01-11 11:11:10 +0100913 rtc::CritScope lock(&local_crit_sect_);
914 return last_framerate_;
915 }
916
Erik Språngd7329ca2019-02-21 21:19:53 +0100917 VideoFrame::UpdateRect GetLastUpdateRect() const {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100918 rtc::CritScope lock(&local_crit_sect_);
919 return last_update_rect_;
920 }
921
Niels Möller87e2d782019-03-07 10:18:23 +0100922 const std::vector<VideoFrameType>& LastFrameTypes() const {
Erik Språngd7329ca2019-02-21 21:19:53 +0100923 rtc::CritScope lock(&local_crit_sect_);
924 return last_frame_types_;
925 }
926
927 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100928 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100929 keyframe ? VideoFrameType::kVideoFrameKey
930 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100931 {
932 rtc::CritScope lock(&local_crit_sect_);
933 last_frame_types_ = frame_type;
934 }
Niels Möllerb859b322019-03-07 12:40:01 +0100935 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 }
937
Erik Språngb7cb7b52019-02-26 15:52:33 +0100938 void InjectEncodedImage(const EncodedImage& image) {
939 rtc::CritScope lock(&local_crit_sect_);
940 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
941 }
942
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200943 void InjectEncodedImage(const EncodedImage& image,
944 const CodecSpecificInfo* codec_specific_info,
945 const RTPFragmentationHeader* fragmentation) {
946 rtc::CritScope lock(&local_crit_sect_);
947 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
948 fragmentation);
949 }
950
Erik Språngd7329ca2019-02-21 21:19:53 +0100951 void ExpectNullFrame() {
952 rtc::CritScope lock(&local_crit_sect_);
953 expect_null_frame_ = true;
954 }
955
Erik Språng5056af02019-09-02 15:53:11 +0200956 absl::optional<VideoEncoder::RateControlParameters>
957 GetAndResetLastRateControlSettings() {
958 auto settings = last_rate_control_settings_;
959 last_rate_control_settings_.reset();
960 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100961 }
962
Sergey Silkin5ee69672019-07-02 14:18:34 +0200963 int GetNumEncoderInitializations() const {
964 rtc::CritScope lock(&local_crit_sect_);
965 return num_encoder_initializations_;
966 }
967
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200968 int GetNumSetRates() const {
969 rtc::CritScope lock(&local_crit_sect_);
970 return num_set_rates_;
971 }
972
perkjfa10b552016-10-02 23:45:26 -0700973 private:
perkj26091b12016-09-01 01:17:40 -0700974 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100975 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700976 bool block_encode;
977 {
brandtre78d2662017-01-16 05:57:16 -0800978 rtc::CritScope lock(&local_crit_sect_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100979 if (expect_null_frame_) {
980 EXPECT_EQ(input_image.timestamp(), 0u);
981 EXPECT_EQ(input_image.width(), 1);
982 last_frame_types_ = *frame_types;
983 expect_null_frame_ = false;
984 } else {
985 EXPECT_GT(input_image.timestamp(), timestamp_);
986 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
987 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
988 }
perkj26091b12016-09-01 01:17:40 -0700989
990 timestamp_ = input_image.timestamp();
991 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700992 last_input_width_ = input_image.width();
993 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700994 block_encode = block_next_encode_;
995 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100996 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100997 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700998 }
Niels Möllerb859b322019-03-07 12:40:01 +0100999 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001000 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001001 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001002 return result;
1003 }
1004
sprangfe627f32017-03-29 08:24:59 -07001005 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001006 const Settings& settings) override {
1007 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001008
sprangfe627f32017-03-29 08:24:59 -07001009 rtc::CritScope lock(&local_crit_sect_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001010 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001011
1012 ++num_encoder_initializations_;
1013
Erik Språng82fad3d2018-03-21 09:57:23 +01001014 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001015 // Simulate setting up temporal layers, in order to validate the life
1016 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001017 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001018 frame_buffer_controller_ =
1019 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001020 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001021 if (force_init_encode_failed_) {
1022 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001023 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001024 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001025
Erik Språngb7cb7b52019-02-26 15:52:33 +01001026 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001027 return res;
1028 }
1029
Erik Språngb7cb7b52019-02-26 15:52:33 +01001030 int32_t Release() override {
1031 rtc::CritScope lock(&local_crit_sect_);
1032 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1033 initialized_ = EncoderState::kUninitialized;
1034 return FakeEncoder::Release();
1035 }
1036
Erik Språng16cb8f52019-04-12 13:59:09 +02001037 void SetRates(const RateControlParameters& parameters) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001038 rtc::CritScope lock(&local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001039 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001040 VideoBitrateAllocation adjusted_rate_allocation;
1041 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1042 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001043 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001044 adjusted_rate_allocation.SetBitrate(
1045 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001046 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001047 rate_factor_));
1048 }
1049 }
1050 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001051 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001052 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001053 RateControlParameters adjusted_paramters = parameters;
1054 adjusted_paramters.bitrate = adjusted_rate_allocation;
1055 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001056 }
1057
brandtre78d2662017-01-16 05:57:16 -08001058 rtc::CriticalSection local_crit_sect_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001059 enum class EncoderState {
1060 kUninitialized,
1061 kInitializationFailed,
1062 kInitialized
1063 } initialized_ RTC_GUARDED_BY(local_crit_sect_) =
1064 EncoderState::kUninitialized;
danilchapa37de392017-09-09 04:17:22 -07001065 bool block_next_encode_ RTC_GUARDED_BY(local_crit_sect_) = false;
perkj26091b12016-09-01 01:17:40 -07001066 rtc::Event continue_encode_event_;
danilchapa37de392017-09-09 04:17:22 -07001067 uint32_t timestamp_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1068 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1069 int last_input_width_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1070 int last_input_height_ RTC_GUARDED_BY(local_crit_sect_) = 0;
1071 bool quality_scaling_ RTC_GUARDED_BY(local_crit_sect_) = true;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001072 int requested_resolution_alignment_ RTC_GUARDED_BY(local_crit_sect_) = 1;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001073 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_crit_sect_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +01001074 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
danilchapa37de392017-09-09 04:17:22 -07001075 RTC_GUARDED_BY(local_crit_sect_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001076 absl::optional<bool>
1077 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
1078 local_crit_sect_);
danilchapa37de392017-09-09 04:17:22 -07001079 bool force_init_encode_failed_ RTC_GUARDED_BY(local_crit_sect_) = false;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001080 double rate_factor_ RTC_GUARDED_BY(local_crit_sect_) = 1.0;
1081 uint32_t last_framerate_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001082 absl::optional<VideoEncoder::RateControlParameters>
1083 last_rate_control_settings_;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001084 VideoFrame::UpdateRect last_update_rect_
1085 RTC_GUARDED_BY(local_crit_sect_) = {0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001086 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001087 bool expect_null_frame_ = false;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001088 EncodedImageCallback* encoded_image_callback_
1089 RTC_GUARDED_BY(local_crit_sect_) = nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001090 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001091 int num_encoder_initializations_ RTC_GUARDED_BY(local_crit_sect_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001092 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
1093 RTC_GUARDED_BY(local_crit_sect_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001094 int num_set_rates_ RTC_GUARDED_BY(local_crit_sect_) = 0;
perkj26091b12016-09-01 01:17:40 -07001095 };
1096
mflodmancc3d4422017-08-03 08:27:51 -07001097 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001098 public:
1099 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001100 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001101
perkj26091b12016-09-01 01:17:40 -07001102 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001103 EXPECT_TRUE(
1104 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1105 }
1106
1107 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1108 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001109 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001110 if (!encoded_frame_event_.Wait(timeout_ms))
1111 return false;
perkj26091b12016-09-01 01:17:40 -07001112 {
1113 rtc::CritScope lock(&crit_);
sprangb1ca0732017-02-01 08:38:12 -08001114 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001115 }
1116 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001117 return true;
perkj26091b12016-09-01 01:17:40 -07001118 }
1119
sprangb1ca0732017-02-01 08:38:12 -08001120 void WaitForEncodedFrame(uint32_t expected_width,
1121 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001122 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001123 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001124 }
1125
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001126 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001127 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001128 uint32_t width = 0;
1129 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001130 {
1131 rtc::CritScope lock(&crit_);
1132 width = last_width_;
1133 height = last_height_;
1134 }
1135 EXPECT_EQ(expected_height, height);
1136 EXPECT_EQ(expected_width, width);
1137 }
1138
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001139 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1140 int width = 0;
1141 int height = 0;
1142 {
1143 rtc::CritScope lock(&crit_);
1144 width = last_width_;
1145 height = last_height_;
1146 }
1147 EXPECT_EQ(width % resolution_alignment, 0);
1148 EXPECT_EQ(height % resolution_alignment, 0);
1149 }
1150
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001151 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1152 VideoRotation rotation;
1153 {
1154 rtc::CritScope lock(&crit_);
1155 rotation = last_rotation_;
1156 }
1157 EXPECT_EQ(expected_rotation, rotation);
1158 }
1159
kthelgason2fc52542017-03-03 00:24:41 -08001160 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001161
sprangc5d62e22017-04-02 23:53:04 -07001162 bool WaitForFrame(int64_t timeout_ms) {
1163 return encoded_frame_event_.Wait(timeout_ms);
1164 }
1165
perkj26091b12016-09-01 01:17:40 -07001166 void SetExpectNoFrames() {
1167 rtc::CritScope lock(&crit_);
1168 expect_frames_ = false;
1169 }
1170
asaperssonfab67072017-04-04 05:51:49 -07001171 int number_of_reconfigurations() const {
Per512ecb32016-09-23 15:52:06 +02001172 rtc::CritScope lock(&crit_);
1173 return number_of_reconfigurations_;
1174 }
1175
asaperssonfab67072017-04-04 05:51:49 -07001176 int last_min_transmit_bitrate() const {
Per512ecb32016-09-23 15:52:06 +02001177 rtc::CritScope lock(&crit_);
1178 return min_transmit_bitrate_bps_;
1179 }
1180
Erik Språngd7329ca2019-02-21 21:19:53 +01001181 void SetNumExpectedLayers(size_t num_layers) {
1182 rtc::CritScope lock(&crit_);
1183 num_expected_layers_ = num_layers;
1184 }
1185
Erik Språngb7cb7b52019-02-26 15:52:33 +01001186 int64_t GetLastCaptureTimeMs() const {
1187 rtc::CritScope lock(&crit_);
1188 return last_capture_time_ms_;
1189 }
1190
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001191 std::vector<uint8_t> GetLastEncodedImageData() {
1192 rtc::CritScope lock(&crit_);
1193 return std::move(last_encoded_image_data_);
1194 }
1195
1196 RTPFragmentationHeader GetLastFragmentation() {
1197 rtc::CritScope lock(&crit_);
1198 return std::move(last_fragmentation_);
1199 }
1200
perkj26091b12016-09-01 01:17:40 -07001201 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001202 Result OnEncodedImage(
1203 const EncodedImage& encoded_image,
1204 const CodecSpecificInfo* codec_specific_info,
1205 const RTPFragmentationHeader* fragmentation) override {
Per512ecb32016-09-23 15:52:06 +02001206 rtc::CritScope lock(&crit_);
1207 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001208 last_encoded_image_data_ = std::vector<uint8_t>(
1209 encoded_image.data(), encoded_image.data() + encoded_image.size());
1210 if (fragmentation) {
1211 last_fragmentation_.CopyFrom(*fragmentation);
1212 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001213 uint32_t timestamp = encoded_image.Timestamp();
1214 if (last_timestamp_ != timestamp) {
1215 num_received_layers_ = 1;
1216 } else {
1217 ++num_received_layers_;
1218 }
1219 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001220 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001221 last_width_ = encoded_image._encodedWidth;
1222 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001223 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001224 if (num_received_layers_ == num_expected_layers_) {
1225 encoded_frame_event_.Set();
1226 }
sprangb1ca0732017-02-01 08:38:12 -08001227 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001228 }
1229
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001230 void OnEncoderConfigurationChanged(
1231 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001232 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001233 VideoEncoderConfig::ContentType content_type,
1234 int min_transmit_bitrate_bps) override {
Sergey Silkin5ee69672019-07-02 14:18:34 +02001235 rtc::CritScope lock(&crit_);
Per512ecb32016-09-23 15:52:06 +02001236 ++number_of_reconfigurations_;
1237 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1238 }
1239
perkj26091b12016-09-01 01:17:40 -07001240 rtc::CriticalSection crit_;
1241 TestEncoder* test_encoder_;
1242 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001243 std::vector<uint8_t> last_encoded_image_data_;
1244 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001245 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001246 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001247 uint32_t last_height_ = 0;
1248 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001249 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001250 size_t num_expected_layers_ = 1;
1251 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001252 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001253 int number_of_reconfigurations_ = 0;
1254 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001255 };
1256
Sergey Silkin5ee69672019-07-02 14:18:34 +02001257 class VideoBitrateAllocatorProxyFactory
1258 : public VideoBitrateAllocatorFactory {
1259 public:
1260 VideoBitrateAllocatorProxyFactory()
1261 : bitrate_allocator_factory_(
1262 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1263
1264 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1265 const VideoCodec& codec) override {
1266 rtc::CritScope lock(&crit_);
1267 codec_config_ = codec;
1268 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1269 }
1270
1271 VideoCodec codec_config() const {
1272 rtc::CritScope lock(&crit_);
1273 return codec_config_;
1274 }
1275
1276 private:
1277 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1278
1279 rtc::CriticalSection crit_;
1280 VideoCodec codec_config_ RTC_GUARDED_BY(crit_);
1281 };
1282
perkj26091b12016-09-01 01:17:40 -07001283 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001284 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001285 int codec_width_;
1286 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001287 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001288 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001289 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001290 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001291 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001292 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001293 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001294 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001295 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001296 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001297};
1298
mflodmancc3d4422017-08-03 08:27:51 -07001299TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001300 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001301 DataRate::BitsPerSec(kTargetBitrateBps),
1302 DataRate::BitsPerSec(kTargetBitrateBps),
1303 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001304 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001305 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001306 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001307 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001308 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001309}
1310
mflodmancc3d4422017-08-03 08:27:51 -07001311TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001312 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001313 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001314 // The encoder will cache up to one frame for a short duration. Adding two
1315 // frames means that the first frame will be dropped and the second frame will
1316 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001317 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001318 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001319 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001320
Henrik Boström381d1092020-05-12 18:49:07 +02001321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001322 DataRate::BitsPerSec(kTargetBitrateBps),
1323 DataRate::BitsPerSec(kTargetBitrateBps),
1324 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001325
Sebastian Janssona3177052018-04-10 13:05:49 +02001326 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001327 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001328 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1329
1330 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001331 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001332}
1333
mflodmancc3d4422017-08-03 08:27:51 -07001334TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001336 DataRate::BitsPerSec(kTargetBitrateBps),
1337 DataRate::BitsPerSec(kTargetBitrateBps),
1338 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001339 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001340 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001341
Henrik Boström381d1092020-05-12 18:49:07 +02001342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1343 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1344 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001345 // The encoder will cache up to one frame for a short duration. Adding two
1346 // frames means that the first frame will be dropped and the second frame will
1347 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001348 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001349 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001350
Henrik Boström381d1092020-05-12 18:49:07 +02001351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001352 DataRate::BitsPerSec(kTargetBitrateBps),
1353 DataRate::BitsPerSec(kTargetBitrateBps),
1354 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001355 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001356 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1357 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001358 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001359}
1360
mflodmancc3d4422017-08-03 08:27:51 -07001361TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001362 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001363 DataRate::BitsPerSec(kTargetBitrateBps),
1364 DataRate::BitsPerSec(kTargetBitrateBps),
1365 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001366 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001367 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001368
1369 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001370 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001371
perkja49cbd32016-09-16 07:53:41 -07001372 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001373 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001374 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001375}
1376
mflodmancc3d4422017-08-03 08:27:51 -07001377TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001378 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001379 DataRate::BitsPerSec(kTargetBitrateBps),
1380 DataRate::BitsPerSec(kTargetBitrateBps),
1381 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001382
perkja49cbd32016-09-16 07:53:41 -07001383 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001384 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001385
mflodmancc3d4422017-08-03 08:27:51 -07001386 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001387 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001388 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001389 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1390 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001391}
1392
mflodmancc3d4422017-08-03 08:27:51 -07001393TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001394 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001395 DataRate::BitsPerSec(kTargetBitrateBps),
1396 DataRate::BitsPerSec(kTargetBitrateBps),
1397 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001398
1399 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001400 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001401 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001402 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1403 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001404 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1405 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001406 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001407 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001408
mflodmancc3d4422017-08-03 08:27:51 -07001409 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001410}
1411
Noah Richards51db4212019-06-12 06:59:12 -07001412TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001414 DataRate::BitsPerSec(kTargetBitrateBps),
1415 DataRate::BitsPerSec(kTargetBitrateBps),
1416 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001417
1418 rtc::Event frame_destroyed_event;
1419 video_source_.IncomingCapturedFrame(
1420 CreateFakeNativeFrame(1, &frame_destroyed_event));
1421 ExpectDroppedFrame();
1422 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1423 video_stream_encoder_->Stop();
1424}
1425
1426TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1427 // Use the cropping factory.
1428 video_encoder_config_.video_stream_factory =
1429 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1430 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1431 kMaxPayloadLength);
1432 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1433
1434 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps),
1438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001439 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1440 WaitForEncodedFrame(1);
1441 // The encoder will have been configured once.
1442 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1443 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1444 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1445
1446 // Now send in a fake frame that needs to be cropped as the width/height
1447 // aren't divisible by 4 (see CreateEncoderStreams above).
1448 rtc::Event frame_destroyed_event;
1449 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1450 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1451 ExpectDroppedFrame();
1452 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1453 video_stream_encoder_->Stop();
1454}
1455
Ying Wang9b881ab2020-02-07 14:29:32 +01001456TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001458 DataRate::BitsPerSec(kTargetBitrateBps),
1459 DataRate::BitsPerSec(kTargetBitrateBps),
1460 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001461 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1462 WaitForEncodedFrame(1);
1463
Henrik Boström381d1092020-05-12 18:49:07 +02001464 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001465 DataRate::BitsPerSec(kTargetBitrateBps),
1466 DataRate::BitsPerSec(kTargetBitrateBps),
1467 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001468 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1469 // frames. Adding two frames means that the first frame will be dropped and
1470 // the second frame will be sent to the encoder.
1471 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1472 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1473 WaitForEncodedFrame(3);
1474 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1475 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1476 WaitForEncodedFrame(5);
1477 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1478 video_stream_encoder_->Stop();
1479}
1480
mflodmancc3d4422017-08-03 08:27:51 -07001481TEST_F(VideoStreamEncoderTest,
1482 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001484 DataRate::BitsPerSec(kTargetBitrateBps),
1485 DataRate::BitsPerSec(kTargetBitrateBps),
1486 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001487 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001488
1489 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001490 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001491 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001492 // The encoder will have been configured once when the first frame is
1493 // received.
1494 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001495
1496 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001497 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001498 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001499 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001500 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001501
1502 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001503 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001504 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001505 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001506 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001507
mflodmancc3d4422017-08-03 08:27:51 -07001508 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001509}
1510
mflodmancc3d4422017-08-03 08:27:51 -07001511TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001513 DataRate::BitsPerSec(kTargetBitrateBps),
1514 DataRate::BitsPerSec(kTargetBitrateBps),
1515 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001516
1517 // Capture a frame and wait for it to synchronize with the encoder thread.
1518 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001519 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001520 // The encoder will have been configured once.
1521 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001522 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1523 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1524
1525 codec_width_ *= 2;
1526 codec_height_ *= 2;
1527 // Capture a frame with a higher resolution and wait for it to synchronize
1528 // with the encoder thread.
1529 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001530 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001531 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1532 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001533 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001534
mflodmancc3d4422017-08-03 08:27:51 -07001535 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001536}
1537
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001538TEST_F(VideoStreamEncoderTest,
1539 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001541 DataRate::BitsPerSec(kTargetBitrateBps),
1542 DataRate::BitsPerSec(kTargetBitrateBps),
1543 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001544
1545 // Capture a frame and wait for it to synchronize with the encoder thread.
1546 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1547 WaitForEncodedFrame(1);
1548
1549 VideoEncoderConfig video_encoder_config;
1550 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1551 // Changing the max payload data length recreates encoder.
1552 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1553 kMaxPayloadLength / 2);
1554
1555 // Capture a frame and wait for it to synchronize with the encoder thread.
1556 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1557 WaitForEncodedFrame(2);
1558 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1559
1560 video_stream_encoder_->Stop();
1561}
1562
Sergey Silkin5ee69672019-07-02 14:18:34 +02001563TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001564 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001565 DataRate::BitsPerSec(kTargetBitrateBps),
1566 DataRate::BitsPerSec(kTargetBitrateBps),
1567 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001568
1569 VideoEncoderConfig video_encoder_config;
1570 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1571 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1572 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1573 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1574 kMaxPayloadLength);
1575
1576 // Capture a frame and wait for it to synchronize with the encoder thread.
1577 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1578 WaitForEncodedFrame(1);
1579 // The encoder will have been configured once when the first frame is
1580 // received.
1581 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1582 EXPECT_EQ(kTargetBitrateBps,
1583 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1584 EXPECT_EQ(kStartBitrateBps,
1585 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1586
Sergey Silkin6456e352019-07-08 17:56:40 +02001587 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1588 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001589 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1590 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1591 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1592 kMaxPayloadLength);
1593
1594 // Capture a frame and wait for it to synchronize with the encoder thread.
1595 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1596 WaitForEncodedFrame(2);
1597 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1598 // Bitrate limits have changed - rate allocator should be reconfigured,
1599 // encoder should not be reconfigured.
1600 EXPECT_EQ(kTargetBitrateBps * 2,
1601 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1602 EXPECT_EQ(kStartBitrateBps * 2,
1603 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1604 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1605
1606 video_stream_encoder_->Stop();
1607}
1608
Sergey Silkin6456e352019-07-08 17:56:40 +02001609TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001610 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001612 DataRate::BitsPerSec(kTargetBitrateBps),
1613 DataRate::BitsPerSec(kTargetBitrateBps),
1614 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001615
Sergey Silkincd02eba2020-01-20 14:48:40 +01001616 const uint32_t kMinEncBitrateKbps = 100;
1617 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001618 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001619 /*frame_size_pixels=*/codec_width_ * codec_height_,
1620 /*min_start_bitrate_bps=*/0,
1621 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1622 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001623 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1624
Sergey Silkincd02eba2020-01-20 14:48:40 +01001625 VideoEncoderConfig video_encoder_config;
1626 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1627 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1628 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1629 (kMinEncBitrateKbps + 1) * 1000;
1630 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1631 kMaxPayloadLength);
1632
1633 // When both encoder and app provide bitrate limits, the intersection of
1634 // provided sets should be used.
1635 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1636 WaitForEncodedFrame(1);
1637 EXPECT_EQ(kMaxEncBitrateKbps,
1638 bitrate_allocator_factory_.codec_config().maxBitrate);
1639 EXPECT_EQ(kMinEncBitrateKbps + 1,
1640 bitrate_allocator_factory_.codec_config().minBitrate);
1641
1642 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1643 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1644 (kMinEncBitrateKbps - 1) * 1000;
1645 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1646 kMaxPayloadLength);
1647 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001648 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001649 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001650 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001651 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001652 bitrate_allocator_factory_.codec_config().minBitrate);
1653
Sergey Silkincd02eba2020-01-20 14:48:40 +01001654 video_stream_encoder_->Stop();
1655}
1656
1657TEST_F(VideoStreamEncoderTest,
1658 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001660 DataRate::BitsPerSec(kTargetBitrateBps),
1661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001663
1664 const uint32_t kMinAppBitrateKbps = 100;
1665 const uint32_t kMaxAppBitrateKbps = 200;
1666 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1667 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1668 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1669 /*frame_size_pixels=*/codec_width_ * codec_height_,
1670 /*min_start_bitrate_bps=*/0,
1671 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1672 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1673 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1674
1675 VideoEncoderConfig video_encoder_config;
1676 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1677 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1678 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1679 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001680 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1681 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001682
Sergey Silkincd02eba2020-01-20 14:48:40 +01001683 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1684 WaitForEncodedFrame(1);
1685 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001686 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001687 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001688 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001689
1690 video_stream_encoder_->Stop();
1691}
1692
1693TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001694 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001696 DataRate::BitsPerSec(kTargetBitrateBps),
1697 DataRate::BitsPerSec(kTargetBitrateBps),
1698 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001699
1700 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001701 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001702 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001703 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001704 fake_encoder_.SetResolutionBitrateLimits(
1705 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1706
1707 VideoEncoderConfig video_encoder_config;
1708 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1709 video_encoder_config.max_bitrate_bps = 0;
1710 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1711 kMaxPayloadLength);
1712
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001713 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001714 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1715 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001716 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1717 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001718 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1719 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1720
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001721 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001722 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1723 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001724 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1725 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001726 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1727 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1728
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001729 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001730 // encoder for 360p should be used.
1731 video_source_.IncomingCapturedFrame(
1732 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1733 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001734 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1735 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001736 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1737 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1738
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001739 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001740 // ignored.
1741 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1742 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001743 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1744 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001745 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1746 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001747 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1748 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001749 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1750 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1751
1752 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1753 // for 270p should be used.
1754 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1755 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001756 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1757 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001758 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1759 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1760
1761 video_stream_encoder_->Stop();
1762}
1763
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001764TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001766 DataRate::BitsPerSec(kTargetBitrateBps),
1767 DataRate::BitsPerSec(kTargetBitrateBps),
1768 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001769
1770 VideoEncoderConfig video_encoder_config;
1771 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1772 video_encoder_config.max_bitrate_bps = 0;
1773 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1774 kMaxPayloadLength);
1775
1776 // Encode 720p frame to get the default encoder target bitrate.
1777 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1778 WaitForEncodedFrame(1);
1779 const uint32_t kDefaultTargetBitrateFor720pKbps =
1780 bitrate_allocator_factory_.codec_config()
1781 .simulcastStream[0]
1782 .targetBitrate;
1783
1784 // Set the max recommended encoder bitrate to something lower than the default
1785 // target bitrate.
1786 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1787 1280 * 720, 10 * 1000, 10 * 1000,
1788 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1789 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1790
1791 // Change resolution to trigger encoder reinitialization.
1792 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1793 WaitForEncodedFrame(2);
1794 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1795 WaitForEncodedFrame(3);
1796
1797 // Ensure the target bitrate is capped by the max bitrate.
1798 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1799 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1800 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1801 .simulcastStream[0]
1802 .targetBitrate *
1803 1000,
1804 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1805
1806 video_stream_encoder_->Stop();
1807}
1808
mflodmancc3d4422017-08-03 08:27:51 -07001809TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001810 EXPECT_TRUE(video_source_.has_sinks());
1811 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001812 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001813 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001814 EXPECT_FALSE(video_source_.has_sinks());
1815 EXPECT_TRUE(new_video_source.has_sinks());
1816
mflodmancc3d4422017-08-03 08:27:51 -07001817 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001818}
1819
mflodmancc3d4422017-08-03 08:27:51 -07001820TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001821 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001822 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001823 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001824 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001825}
1826
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001827TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1828 constexpr int kRequestedResolutionAlignment = 7;
1829 video_source_.set_adaptation_enabled(true);
1830 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001832 DataRate::BitsPerSec(kTargetBitrateBps),
1833 DataRate::BitsPerSec(kTargetBitrateBps),
1834 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001835
1836 // On the 1st frame, we should have initialized the encoder and
1837 // asked for its resolution requirements.
1838 video_source_.IncomingCapturedFrame(
1839 CreateFrame(1, codec_width_, codec_height_));
1840 WaitForEncodedFrame(1);
1841 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1842 kRequestedResolutionAlignment);
1843
1844 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1845 // (It's up the to the encoder to potentially drop the previous frame,
1846 // to avoid coding back-to-back keyframes.)
1847 video_source_.IncomingCapturedFrame(
1848 CreateFrame(2, codec_width_, codec_height_));
1849 WaitForEncodedFrame(2);
1850 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1851
1852 video_stream_encoder_->Stop();
1853}
1854
Jonathan Yubc771b72017-12-08 17:04:29 -08001855TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1856 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001857 const int kWidth = 1280;
1858 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001859
1860 // We rely on the automatic resolution adaptation, but we handle framerate
1861 // adaptation manually by mocking the stats proxy.
1862 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001863
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001864 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001866 DataRate::BitsPerSec(kTargetBitrateBps),
1867 DataRate::BitsPerSec(kTargetBitrateBps),
1868 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001869 video_stream_encoder_->SetSource(&video_source_,
1870 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001871 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001872 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001873 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001874 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1875
Jonathan Yubc771b72017-12-08 17:04:29 -08001876 // Adapt down as far as possible.
1877 rtc::VideoSinkWants last_wants;
1878 int64_t t = 1;
1879 int loop_count = 0;
1880 do {
1881 ++loop_count;
1882 last_wants = video_source_.sink_wants();
1883
1884 // Simulate the framerate we've been asked to adapt to.
1885 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1886 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1887 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1888 mock_stats.input_frame_rate = fps;
1889 stats_proxy_->SetMockStats(mock_stats);
1890
1891 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1892 sink_.WaitForEncodedFrame(t);
1893 t += frame_interval_ms;
1894
mflodmancc3d4422017-08-03 08:27:51 -07001895 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001896 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001897 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001898 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1899 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001900 } while (video_source_.sink_wants().max_pixel_count <
1901 last_wants.max_pixel_count ||
1902 video_source_.sink_wants().max_framerate_fps <
1903 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001904
Jonathan Yubc771b72017-12-08 17:04:29 -08001905 // Verify that we've adapted all the way down.
1906 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001907 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001908 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1909 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001910 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001911 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1912 *video_source_.last_sent_height());
1913 EXPECT_EQ(kMinBalancedFramerateFps,
1914 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001915
Jonathan Yubc771b72017-12-08 17:04:29 -08001916 // Adapt back up the same number of times we adapted down.
1917 for (int i = 0; i < loop_count - 1; ++i) {
1918 last_wants = video_source_.sink_wants();
1919
1920 // Simulate the framerate we've been asked to adapt to.
1921 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1922 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1923 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1924 mock_stats.input_frame_rate = fps;
1925 stats_proxy_->SetMockStats(mock_stats);
1926
1927 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1928 sink_.WaitForEncodedFrame(t);
1929 t += frame_interval_ms;
1930
Henrik Boström91aa7322020-04-28 12:24:33 +02001931 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001932 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001933 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001934 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1935 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001936 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1937 last_wants.max_pixel_count ||
1938 video_source_.sink_wants().max_framerate_fps >
1939 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001940 }
1941
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001942 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001943 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001944 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001945 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1946 EXPECT_EQ((loop_count - 1) * 2,
1947 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001948
mflodmancc3d4422017-08-03 08:27:51 -07001949 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001950}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001951
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001952TEST_F(VideoStreamEncoderTest,
1953 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1954 video_stream_encoder_->OnBitrateUpdated(
1955 DataRate::BitsPerSec(kTargetBitrateBps),
1956 DataRate::BitsPerSec(kTargetBitrateBps),
1957 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001958 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001959
1960 const int kFrameWidth = 1280;
1961 const int kFrameHeight = 720;
1962
1963 int64_t ntp_time = kFrameIntervalMs;
1964
1965 // Force an input frame rate to be available, or the adaptation call won't
1966 // know what framerate to adapt form.
1967 const int kInputFps = 30;
1968 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1969 stats.input_frame_rate = kInputFps;
1970 stats_proxy_->SetMockStats(stats);
1971
1972 video_source_.set_adaptation_enabled(true);
1973 video_stream_encoder_->SetSource(
1974 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001975 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001976 video_source_.IncomingCapturedFrame(
1977 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1978 sink_.WaitForEncodedFrame(ntp_time);
1979 ntp_time += kFrameIntervalMs;
1980
1981 // Trigger CPU overuse.
1982 video_stream_encoder_->TriggerCpuOveruse();
1983 video_source_.IncomingCapturedFrame(
1984 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1985 sink_.WaitForEncodedFrame(ntp_time);
1986 ntp_time += kFrameIntervalMs;
1987
1988 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1989 EXPECT_EQ(std::numeric_limits<int>::max(),
1990 video_source_.sink_wants().max_pixel_count);
1991 // Some framerate constraint should be set.
1992 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1993 EXPECT_LT(restricted_fps, kInputFps);
1994 video_source_.IncomingCapturedFrame(
1995 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1996 sink_.WaitForEncodedFrame(ntp_time);
1997 ntp_time += 100;
1998
Henrik Boström2671dac2020-05-19 16:29:09 +02001999 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002000 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2001 // Give the encoder queue time to process the change in degradation preference
2002 // by waiting for an encoded frame.
2003 video_source_.IncomingCapturedFrame(
2004 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2005 sink_.WaitForEncodedFrame(ntp_time);
2006 ntp_time += kFrameIntervalMs;
2007
2008 video_stream_encoder_->TriggerQualityLow();
2009 video_source_.IncomingCapturedFrame(
2010 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2011 sink_.WaitForEncodedFrame(ntp_time);
2012 ntp_time += kFrameIntervalMs;
2013
2014 // Some resolution constraint should be set.
2015 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2016 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2017 kFrameWidth * kFrameHeight);
2018 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2019
2020 int pixel_count = video_source_.sink_wants().max_pixel_count;
2021 // Triggering a CPU underuse should not change the sink wants since it has
2022 // not been overused for resolution since we changed degradation preference.
2023 video_stream_encoder_->TriggerCpuUnderuse();
2024 video_source_.IncomingCapturedFrame(
2025 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2026 sink_.WaitForEncodedFrame(ntp_time);
2027 ntp_time += kFrameIntervalMs;
2028 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2029 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2030
2031 // Change the degradation preference back. CPU underuse should now adapt.
Henrik Boström2671dac2020-05-19 16:29:09 +02002032 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002033 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2034 video_source_.IncomingCapturedFrame(
2035 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2036 sink_.WaitForEncodedFrame(ntp_time);
2037 ntp_time += 100;
2038 // Resolution adaptations is gone after changing degradation preference.
2039 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2040 EXPECT_EQ(std::numeric_limits<int>::max(),
2041 video_source_.sink_wants().max_pixel_count);
2042 // The fps adaptation from above is now back.
2043 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2044
2045 // Trigger CPU underuse.
2046 video_stream_encoder_->TriggerCpuUnderuse();
2047 video_source_.IncomingCapturedFrame(
2048 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2049 sink_.WaitForEncodedFrame(ntp_time);
2050 ntp_time += kFrameIntervalMs;
2051 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2052
2053 video_stream_encoder_->Stop();
2054}
2055
mflodmancc3d4422017-08-03 08:27:51 -07002056TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002058 DataRate::BitsPerSec(kTargetBitrateBps),
2059 DataRate::BitsPerSec(kTargetBitrateBps),
2060 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002061 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002062
sprangc5d62e22017-04-02 23:53:04 -07002063 const int kFrameWidth = 1280;
2064 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002065
Åsa Persson8c1bf952018-09-13 10:42:19 +02002066 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002067
kthelgason5e13d412016-12-01 03:59:51 -08002068 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002069 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002070 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002071 frame_timestamp += kFrameIntervalMs;
2072
perkj803d97f2016-11-01 11:45:46 -07002073 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002074 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002075 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002076 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002077 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002078 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002079
asapersson0944a802017-04-07 00:57:58 -07002080 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002081 // wanted resolution.
2082 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2083 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2084 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002085 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002086
2087 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002088 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002089 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002090 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002091 // Give the encoder queue time to process the change in degradation preference
2092 // by waiting for an encoded frame.
2093 new_video_source.IncomingCapturedFrame(
2094 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2095 sink_.WaitForEncodedFrame(frame_timestamp);
2096 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002097 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002098 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002099
sprangc5d62e22017-04-02 23:53:04 -07002100 // Force an input frame rate to be available, or the adaptation call won't
2101 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002102 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002103 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002104 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002105 stats_proxy_->SetMockStats(stats);
2106
mflodmancc3d4422017-08-03 08:27:51 -07002107 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002108 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002109 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002110 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002111 frame_timestamp += kFrameIntervalMs;
2112
2113 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002114 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002115 EXPECT_EQ(std::numeric_limits<int>::max(),
2116 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002117 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002118
asapersson02465b82017-04-10 01:12:52 -07002119 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002120 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2121 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002122 // Give the encoder queue time to process the change in degradation preference
2123 // by waiting for an encoded frame.
2124 new_video_source.IncomingCapturedFrame(
2125 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2126 sink_.WaitForEncodedFrame(frame_timestamp);
2127 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002128 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002129
mflodmancc3d4422017-08-03 08:27:51 -07002130 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002131 new_video_source.IncomingCapturedFrame(
2132 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002133 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002134 frame_timestamp += kFrameIntervalMs;
2135
2136 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002137 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002138
2139 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002140 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002141 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002142 // Give the encoder queue time to process the change in degradation preference
2143 // by waiting for an encoded frame.
2144 new_video_source.IncomingCapturedFrame(
2145 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2146 sink_.WaitForEncodedFrame(frame_timestamp);
2147 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002148 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2149 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002150 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002151 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002152
2153 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002154 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002155 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002156 // Give the encoder queue time to process the change in degradation preference
2157 // by waiting for an encoded frame.
2158 new_video_source.IncomingCapturedFrame(
2159 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2160 sink_.WaitForEncodedFrame(frame_timestamp);
2161 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002162 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2163 EXPECT_EQ(std::numeric_limits<int>::max(),
2164 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002165 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002166
mflodmancc3d4422017-08-03 08:27:51 -07002167 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002168}
2169
mflodmancc3d4422017-08-03 08:27:51 -07002170TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002171 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002172 DataRate::BitsPerSec(kTargetBitrateBps),
2173 DataRate::BitsPerSec(kTargetBitrateBps),
2174 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002175
asaperssonfab67072017-04-04 05:51:49 -07002176 const int kWidth = 1280;
2177 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002178 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002179 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002180 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2181 EXPECT_FALSE(stats.bw_limited_resolution);
2182 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2183
2184 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002185 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002186 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002187 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002188
2189 stats = stats_proxy_->GetStats();
2190 EXPECT_TRUE(stats.bw_limited_resolution);
2191 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2192
2193 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002194 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002195 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002196 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002197
2198 stats = stats_proxy_->GetStats();
2199 EXPECT_FALSE(stats.bw_limited_resolution);
2200 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2201 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2202
mflodmancc3d4422017-08-03 08:27:51 -07002203 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002204}
2205
mflodmancc3d4422017-08-03 08:27:51 -07002206TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002207 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002208 DataRate::BitsPerSec(kTargetBitrateBps),
2209 DataRate::BitsPerSec(kTargetBitrateBps),
2210 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002211
2212 const int kWidth = 1280;
2213 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002214 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002215 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002216 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2217 EXPECT_FALSE(stats.cpu_limited_resolution);
2218 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2219
2220 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002221 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002222 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002223 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002224
2225 stats = stats_proxy_->GetStats();
2226 EXPECT_TRUE(stats.cpu_limited_resolution);
2227 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2228
2229 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002230 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002231 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002232 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002233
2234 stats = stats_proxy_->GetStats();
2235 EXPECT_FALSE(stats.cpu_limited_resolution);
2236 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002237 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002238
mflodmancc3d4422017-08-03 08:27:51 -07002239 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002240}
2241
mflodmancc3d4422017-08-03 08:27:51 -07002242TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002244 DataRate::BitsPerSec(kTargetBitrateBps),
2245 DataRate::BitsPerSec(kTargetBitrateBps),
2246 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002247
asaperssonfab67072017-04-04 05:51:49 -07002248 const int kWidth = 1280;
2249 const int kHeight = 720;
2250 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002251 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002252 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002253 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002254 EXPECT_FALSE(stats.cpu_limited_resolution);
2255 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2256
asaperssonfab67072017-04-04 05:51:49 -07002257 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002258 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002259 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002260 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002261 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002262 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002263 EXPECT_TRUE(stats.cpu_limited_resolution);
2264 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2265
2266 // Set new source with adaptation still enabled.
2267 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002268 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002269 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002270
asaperssonfab67072017-04-04 05:51:49 -07002271 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002272 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002273 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002274 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002275 EXPECT_TRUE(stats.cpu_limited_resolution);
2276 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2277
2278 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002279 video_stream_encoder_->SetSource(&new_video_source,
2280 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002281
asaperssonfab67072017-04-04 05:51:49 -07002282 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002283 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002284 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002285 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002286 EXPECT_FALSE(stats.cpu_limited_resolution);
2287 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2288
2289 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002290 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002291 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002292
asaperssonfab67072017-04-04 05:51:49 -07002293 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002294 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002295 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002296 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002297 EXPECT_TRUE(stats.cpu_limited_resolution);
2298 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2299
asaperssonfab67072017-04-04 05:51:49 -07002300 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002301 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002302 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002303 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002304 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002305 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002306 EXPECT_FALSE(stats.cpu_limited_resolution);
2307 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002308 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002309
mflodmancc3d4422017-08-03 08:27:51 -07002310 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002311}
2312
mflodmancc3d4422017-08-03 08:27:51 -07002313TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002314 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002315 DataRate::BitsPerSec(kTargetBitrateBps),
2316 DataRate::BitsPerSec(kTargetBitrateBps),
2317 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002318
asaperssonfab67072017-04-04 05:51:49 -07002319 const int kWidth = 1280;
2320 const int kHeight = 720;
2321 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002322 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002323 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002324 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002325 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002326 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002327
2328 // Set new source with adaptation still enabled.
2329 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002330 video_stream_encoder_->SetSource(&new_video_source,
2331 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002332
asaperssonfab67072017-04-04 05:51:49 -07002333 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002334 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002335 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002336 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002337 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002338 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002339
asaperssonfab67072017-04-04 05:51:49 -07002340 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002341 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002342 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002343 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002344 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002345 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002346 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002347 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002348
asaperssonfab67072017-04-04 05:51:49 -07002349 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002350 video_stream_encoder_->SetSource(&new_video_source,
2351 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002352
asaperssonfab67072017-04-04 05:51:49 -07002353 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002354 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002355 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002356 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002357 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002358 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002359
asapersson02465b82017-04-10 01:12:52 -07002360 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002361 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002362 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002363
asaperssonfab67072017-04-04 05:51:49 -07002364 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002365 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002366 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002367 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002368 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002369 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2370 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002371
mflodmancc3d4422017-08-03 08:27:51 -07002372 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002373}
2374
mflodmancc3d4422017-08-03 08:27:51 -07002375TEST_F(VideoStreamEncoderTest,
2376 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002377 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002378 DataRate::BitsPerSec(kTargetBitrateBps),
2379 DataRate::BitsPerSec(kTargetBitrateBps),
2380 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002381
2382 const int kWidth = 1280;
2383 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002384 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002385 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002386 video_source_.IncomingCapturedFrame(
2387 CreateFrame(timestamp_ms, kWidth, kHeight));
2388 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002389 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2391 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2392
2393 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002394 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002395 timestamp_ms += kFrameIntervalMs;
2396 video_source_.IncomingCapturedFrame(
2397 CreateFrame(timestamp_ms, kWidth, kHeight));
2398 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002399 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2401 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2402
2403 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002404 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002405 timestamp_ms += kFrameIntervalMs;
2406 video_source_.IncomingCapturedFrame(
2407 CreateFrame(timestamp_ms, kWidth, kHeight));
2408 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002409 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2410 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2411 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2412
Niels Möller4db138e2018-04-19 09:04:13 +02002413 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002414 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002415
2416 VideoEncoderConfig video_encoder_config;
2417 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2418 // Make format different, to force recreation of encoder.
2419 video_encoder_config.video_format.parameters["foo"] = "foo";
2420 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002421 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002422 timestamp_ms += kFrameIntervalMs;
2423 video_source_.IncomingCapturedFrame(
2424 CreateFrame(timestamp_ms, kWidth, kHeight));
2425 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002426 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2428 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2429
mflodmancc3d4422017-08-03 08:27:51 -07002430 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002431}
2432
mflodmancc3d4422017-08-03 08:27:51 -07002433TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002434 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002436 DataRate::BitsPerSec(kTargetBitrateBps),
2437 DataRate::BitsPerSec(kTargetBitrateBps),
2438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2439
2440 const int kWidth = 1280;
2441 const int kHeight = 720;
2442 int sequence = 1;
2443
2444 // Enable BALANCED preference, no initial limitation.
2445 test::FrameForwarder source;
2446 video_stream_encoder_->SetSource(&source,
2447 webrtc::DegradationPreference::BALANCED);
2448 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2449 WaitForEncodedFrame(sequence++);
2450 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2451 EXPECT_FALSE(stats.cpu_limited_resolution);
2452 EXPECT_FALSE(stats.cpu_limited_framerate);
2453 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2454
2455 // Trigger CPU overuse, should now adapt down.
2456 video_stream_encoder_->TriggerCpuOveruse();
2457 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2458 WaitForEncodedFrame(sequence++);
2459 stats = stats_proxy_->GetStats();
2460 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2461
2462 // Set new degradation preference should clear restrictions since we changed
2463 // from BALANCED.
2464 video_stream_encoder_->SetSource(
2465 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2466 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2467 WaitForEncodedFrame(sequence++);
2468 stats = stats_proxy_->GetStats();
2469 EXPECT_FALSE(stats.cpu_limited_resolution);
2470 EXPECT_FALSE(stats.cpu_limited_framerate);
2471 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2472
2473 // Force an input frame rate to be available, or the adaptation call won't
2474 // know what framerate to adapt from.
2475 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2476 mock_stats.input_frame_rate = 30;
2477 stats_proxy_->SetMockStats(mock_stats);
2478 video_stream_encoder_->TriggerCpuOveruse();
2479 stats_proxy_->ResetMockStats();
2480 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2481 WaitForEncodedFrame(sequence++);
2482
2483 // We have now adapted once.
2484 stats = stats_proxy_->GetStats();
2485 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2486
2487 // Back to BALANCED, should clear the restrictions again.
2488 video_stream_encoder_->SetSource(&source,
2489 webrtc::DegradationPreference::BALANCED);
2490 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2491 WaitForEncodedFrame(sequence++);
2492 stats = stats_proxy_->GetStats();
2493 EXPECT_FALSE(stats.cpu_limited_resolution);
2494 EXPECT_FALSE(stats.cpu_limited_framerate);
2495 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2496
2497 video_stream_encoder_->Stop();
2498}
2499
2500TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002501 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002503 DataRate::BitsPerSec(kTargetBitrateBps),
2504 DataRate::BitsPerSec(kTargetBitrateBps),
2505 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002506
asapersson0944a802017-04-07 00:57:58 -07002507 const int kWidth = 1280;
2508 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002509 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002510
asaperssonfab67072017-04-04 05:51:49 -07002511 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002512 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002513 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002514 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002515 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002516 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2517
asapersson02465b82017-04-10 01:12:52 -07002518 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002519 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002520 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002521 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002522 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002523 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002524 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002525 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2526
2527 // Set new source with adaptation still enabled.
2528 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002529 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002530 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002531
2532 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002533 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002534 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002535 stats = stats_proxy_->GetStats();
2536 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002537 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002538 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2539
sprangc5d62e22017-04-02 23:53:04 -07002540 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002541 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002542 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002543 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002544 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002545 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002546 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002547 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002548 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002549 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002550 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2551
sprangc5d62e22017-04-02 23:53:04 -07002552 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002553 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002554 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2555 mock_stats.input_frame_rate = 30;
2556 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002557 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002558 stats_proxy_->ResetMockStats();
2559
2560 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002561 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002562 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002563
2564 // Framerate now adapted.
2565 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002566 EXPECT_FALSE(stats.cpu_limited_resolution);
2567 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002568 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2569
2570 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002571 video_stream_encoder_->SetSource(&new_video_source,
2572 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002573 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002574 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002575 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002576
2577 stats = stats_proxy_->GetStats();
2578 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002579 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002580 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2581
2582 // Try to trigger overuse. Should not succeed.
2583 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002584 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002585 stats_proxy_->ResetMockStats();
2586
2587 stats = stats_proxy_->GetStats();
2588 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002589 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002590 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2591
2592 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002593 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002594 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002595 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002596 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002597 stats = stats_proxy_->GetStats();
2598 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002599 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002600 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002601
2602 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002603 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002604 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002605 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002606 stats = stats_proxy_->GetStats();
2607 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002608 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002609 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2610
2611 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002612 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002613 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002614 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002615 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002616 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002617 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002618 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002619 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002620 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002621 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2622
2623 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002624 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002625 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002626 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002627 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002628 stats = stats_proxy_->GetStats();
2629 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002630 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002631 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002632 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002633
mflodmancc3d4422017-08-03 08:27:51 -07002634 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002635}
2636
mflodmancc3d4422017-08-03 08:27:51 -07002637TEST_F(VideoStreamEncoderTest,
2638 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002639 const int kWidth = 1280;
2640 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002642 DataRate::BitsPerSec(kTargetBitrateBps),
2643 DataRate::BitsPerSec(kTargetBitrateBps),
2644 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002645
asaperssonfab67072017-04-04 05:51:49 -07002646 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002647 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002648
asaperssonfab67072017-04-04 05:51:49 -07002649 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002650 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002651
asaperssonfab67072017-04-04 05:51:49 -07002652 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002653 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002654
asaperssonfab67072017-04-04 05:51:49 -07002655 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002656 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002657
kthelgason876222f2016-11-29 01:44:11 -08002658 // Expect a scale down.
2659 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002660 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002661
asapersson02465b82017-04-10 01:12:52 -07002662 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002663 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002664 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002665 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002666
asaperssonfab67072017-04-04 05:51:49 -07002667 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002668 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002669 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002670 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002671
asaperssonfab67072017-04-04 05:51:49 -07002672 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002673 EXPECT_EQ(std::numeric_limits<int>::max(),
2674 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002675
asaperssonfab67072017-04-04 05:51:49 -07002676 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002677 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002678 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002679 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002680
asapersson02465b82017-04-10 01:12:52 -07002681 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002682 EXPECT_EQ(std::numeric_limits<int>::max(),
2683 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002684
mflodmancc3d4422017-08-03 08:27:51 -07002685 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002686}
2687
mflodmancc3d4422017-08-03 08:27:51 -07002688TEST_F(VideoStreamEncoderTest,
2689 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002690 const int kWidth = 1280;
2691 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002692 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002693 DataRate::BitsPerSec(kTargetBitrateBps),
2694 DataRate::BitsPerSec(kTargetBitrateBps),
2695 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002696
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002697 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002698 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002699 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002700 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002701
2702 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002703 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002704 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002705 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2706 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2707
2708 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002709 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002710 EXPECT_THAT(source.sink_wants(),
2711 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002712 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2713 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2714 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2715
2716 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002717 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002718 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2719 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2720 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2721
mflodmancc3d4422017-08-03 08:27:51 -07002722 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002723}
2724
mflodmancc3d4422017-08-03 08:27:51 -07002725TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002726 const int kWidth = 1280;
2727 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002729 DataRate::BitsPerSec(kTargetBitrateBps),
2730 DataRate::BitsPerSec(kTargetBitrateBps),
2731 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002732
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002733 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002734 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002735 video_stream_encoder_->SetSource(&source,
2736 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002737 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2738 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002739 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002740
2741 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002742 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002743 EXPECT_THAT(source.sink_wants(),
2744 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002745 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2746 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2747 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2748
2749 // Trigger adapt down for same input resolution, expect no change.
2750 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2751 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002752 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002753 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2754 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2756
2757 // Trigger adapt down for larger input resolution, expect no change.
2758 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2759 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002760 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002761 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2762 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2763 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2764
mflodmancc3d4422017-08-03 08:27:51 -07002765 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002766}
2767
mflodmancc3d4422017-08-03 08:27:51 -07002768TEST_F(VideoStreamEncoderTest,
2769 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002770 const int kWidth = 1280;
2771 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002773 DataRate::BitsPerSec(kTargetBitrateBps),
2774 DataRate::BitsPerSec(kTargetBitrateBps),
2775 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002776
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002777 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002778 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002779 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002780 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002781
2782 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002783 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002784 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2787
2788 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002789 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002790 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2793
mflodmancc3d4422017-08-03 08:27:51 -07002794 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002795}
2796
mflodmancc3d4422017-08-03 08:27:51 -07002797TEST_F(VideoStreamEncoderTest,
2798 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002799 const int kWidth = 1280;
2800 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002802 DataRate::BitsPerSec(kTargetBitrateBps),
2803 DataRate::BitsPerSec(kTargetBitrateBps),
2804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002805
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002806 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002807 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002808 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002809 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002810
2811 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002812 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002813 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002814 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002815 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2816
2817 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002818 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002819 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002820 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002821 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2822
mflodmancc3d4422017-08-03 08:27:51 -07002823 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002824}
2825
mflodmancc3d4422017-08-03 08:27:51 -07002826TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002827 const int kWidth = 1280;
2828 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002830 DataRate::BitsPerSec(kTargetBitrateBps),
2831 DataRate::BitsPerSec(kTargetBitrateBps),
2832 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002833
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002834 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002835 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002836 video_stream_encoder_->SetSource(&source,
2837 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002838
2839 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2840 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002841 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2845
2846 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002848 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2850 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2851 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2852
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002854}
2855
mflodmancc3d4422017-08-03 08:27:51 -07002856TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002857 const int kWidth = 1280;
2858 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002860 DataRate::BitsPerSec(kTargetBitrateBps),
2861 DataRate::BitsPerSec(kTargetBitrateBps),
2862 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002863
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002864 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002865 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002866 video_stream_encoder_->SetSource(&source,
2867 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002868
2869 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2870 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002871 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002872 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2873 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2874 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2875
2876 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002878 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002879 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2880 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2881 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2882
mflodmancc3d4422017-08-03 08:27:51 -07002883 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002884}
2885
mflodmancc3d4422017-08-03 08:27:51 -07002886TEST_F(VideoStreamEncoderTest,
2887 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002888 const int kWidth = 1280;
2889 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002891 DataRate::BitsPerSec(kTargetBitrateBps),
2892 DataRate::BitsPerSec(kTargetBitrateBps),
2893 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002894
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002895 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002896 AdaptingFrameForwarder source;
2897 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002898 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002899 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002900
2901 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002902 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002903 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002904 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2905 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2906
2907 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002908 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002909 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002910 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002911 EXPECT_THAT(source.sink_wants(),
2912 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002913 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2914 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2915
2916 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002917 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2920 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2921 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2922
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002924}
2925
mflodmancc3d4422017-08-03 08:27:51 -07002926TEST_F(VideoStreamEncoderTest,
2927 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002928 const int kWidth = 1280;
2929 const int kHeight = 720;
2930 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002932 DataRate::BitsPerSec(kTargetBitrateBps),
2933 DataRate::BitsPerSec(kTargetBitrateBps),
2934 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002935
2936 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2937 stats.input_frame_rate = kInputFps;
2938 stats_proxy_->SetMockStats(stats);
2939
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002940 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002941 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2942 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002943 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002944
2945 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002946 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002947 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2948 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002949 EXPECT_THAT(video_source_.sink_wants(),
2950 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002951
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002952 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002953 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002954 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002955 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002956 // Give the encoder queue time to process the change in degradation preference
2957 // by waiting for an encoded frame.
2958 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2959 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002960 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002961
2962 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002963 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002964 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2965 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002966 EXPECT_THAT(new_video_source.sink_wants(),
2967 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002968
2969 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002970 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002971 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002972
mflodmancc3d4422017-08-03 08:27:51 -07002973 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002974}
2975
mflodmancc3d4422017-08-03 08:27:51 -07002976TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002977 const int kWidth = 1280;
2978 const int kHeight = 720;
2979 const size_t kNumFrames = 10;
2980
Henrik Boström381d1092020-05-12 18:49:07 +02002981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002982 DataRate::BitsPerSec(kTargetBitrateBps),
2983 DataRate::BitsPerSec(kTargetBitrateBps),
2984 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002985
asaperssond0de2952017-04-21 01:47:31 -07002986 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002987 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002988 video_source_.set_adaptation_enabled(true);
2989
2990 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2991 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2992
2993 int downscales = 0;
2994 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002995 video_source_.IncomingCapturedFrame(
2996 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2997 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002998
asaperssonfab67072017-04-04 05:51:49 -07002999 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003000 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003001 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003002 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003003
3004 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3005 ++downscales;
3006
3007 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3008 EXPECT_EQ(downscales,
3009 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3010 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003011 }
mflodmancc3d4422017-08-03 08:27:51 -07003012 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003013}
3014
mflodmancc3d4422017-08-03 08:27:51 -07003015TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003016 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3017 const int kWidth = 1280;
3018 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003020 DataRate::BitsPerSec(kTargetBitrateBps),
3021 DataRate::BitsPerSec(kTargetBitrateBps),
3022 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003023
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003024 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003025 AdaptingFrameForwarder source;
3026 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003027 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003028 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003029
Åsa Persson8c1bf952018-09-13 10:42:19 +02003030 int64_t timestamp_ms = kFrameIntervalMs;
3031 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003032 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003033 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003034 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3035 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3036
3037 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003038 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003039 timestamp_ms += kFrameIntervalMs;
3040 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3041 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003042 EXPECT_THAT(source.sink_wants(),
3043 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003044 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3045 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3046
3047 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003048 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003049 timestamp_ms += kFrameIntervalMs;
3050 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003051 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003052 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003053 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3054 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3055
3056 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003057 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003058 timestamp_ms += kFrameIntervalMs;
3059 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3060 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003061 EXPECT_THAT(source.sink_wants(),
3062 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003063 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3064 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3065
3066 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003067 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003068 timestamp_ms += kFrameIntervalMs;
3069 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003070 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003071 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003072 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3073 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3074
mflodmancc3d4422017-08-03 08:27:51 -07003075 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003076}
3077
mflodmancc3d4422017-08-03 08:27:51 -07003078TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003079 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3080 const int kWidth = 1280;
3081 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003082 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003083 DataRate::BitsPerSec(kTargetBitrateBps),
3084 DataRate::BitsPerSec(kTargetBitrateBps),
3085 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003086
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003087 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003088 AdaptingFrameForwarder source;
3089 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003090 video_stream_encoder_->SetSource(&source,
3091 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003092
Åsa Persson8c1bf952018-09-13 10:42:19 +02003093 int64_t timestamp_ms = kFrameIntervalMs;
3094 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003095 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003096 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003097 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3098 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3099
3100 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003101 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003102 timestamp_ms += kFrameIntervalMs;
3103 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3104 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003105 EXPECT_THAT(source.sink_wants(),
3106 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003107 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3108 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3109
3110 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003111 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003112 timestamp_ms += kFrameIntervalMs;
3113 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003114 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003115 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003116 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3117 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3118
3119 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003120 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003121 timestamp_ms += kFrameIntervalMs;
3122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3123 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003124 EXPECT_THAT(source.sink_wants(),
3125 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003126 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3127 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3128
3129 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003130 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003131 timestamp_ms += kFrameIntervalMs;
3132 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003133 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003134 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003135 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3136 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3137
mflodmancc3d4422017-08-03 08:27:51 -07003138 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003139}
3140
Sergey Silkin41c650b2019-10-14 13:12:19 +02003141TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3142 fake_encoder_.SetResolutionBitrateLimits(
3143 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3144
Henrik Boström381d1092020-05-12 18:49:07 +02003145 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003146 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3147 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3148 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3149 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003150
3151 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3152 AdaptingFrameForwarder source;
3153 source.set_adaptation_enabled(true);
3154 video_stream_encoder_->SetSource(
3155 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3156
3157 // Insert 720p frame.
3158 int64_t timestamp_ms = kFrameIntervalMs;
3159 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3160 WaitForEncodedFrame(1280, 720);
3161
3162 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003163 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003164 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3165 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3166 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3167 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003168 video_stream_encoder_->TriggerQualityLow();
3169
3170 // Insert 720p frame. It should be downscaled and encoded.
3171 timestamp_ms += kFrameIntervalMs;
3172 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3173 WaitForEncodedFrame(960, 540);
3174
3175 // Trigger adapt up. Higher resolution should not be requested duo to lack
3176 // of bitrate.
3177 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003178 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003179
3180 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003181 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003182 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3183 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3184 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3185 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003186
3187 // Trigger adapt up. Higher resolution should be requested.
3188 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003189 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003190
3191 video_stream_encoder_->Stop();
3192}
3193
3194TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3195 fake_encoder_.SetResolutionBitrateLimits(
3196 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3197
3198 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003199 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003200 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3201 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3202 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3203 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003204
3205 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3206 AdaptingFrameForwarder source;
3207 source.set_adaptation_enabled(true);
3208 video_stream_encoder_->SetSource(
3209 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3210
3211 // Insert 720p frame. It should be dropped and lower resolution should be
3212 // requested.
3213 int64_t timestamp_ms = kFrameIntervalMs;
3214 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3215 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003216 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003217
3218 // Insert 720p frame. It should be downscaled and encoded.
3219 timestamp_ms += kFrameIntervalMs;
3220 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3221 WaitForEncodedFrame(960, 540);
3222
3223 video_stream_encoder_->Stop();
3224}
3225
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003226class BalancedDegradationTest : public VideoStreamEncoderTest {
3227 protected:
3228 void SetupTest() {
3229 // Reset encoder for field trials to take effect.
3230 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003231 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003232
3233 // Enable BALANCED preference.
3234 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003235 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3236 }
3237
3238 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003239 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003240 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3241 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003242 }
3243
Åsa Persson45b176f2019-09-30 11:19:05 +02003244 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003245 timestamp_ms_ += kFrameIntervalMs;
3246 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003247 }
3248
3249 void InsertFrameAndWaitForEncoded() {
3250 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003251 sink_.WaitForEncodedFrame(timestamp_ms_);
3252 }
3253
3254 const int kWidth = 640; // pixels:640x360=230400
3255 const int kHeight = 360;
3256 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3257 int64_t timestamp_ms_ = 0;
3258 AdaptingFrameForwarder source_;
3259};
3260
3261TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3262 test::ScopedFieldTrials field_trials(
3263 "WebRTC-Video-BalancedDegradationSettings/"
3264 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3265 SetupTest();
3266
3267 // Force input frame rate.
3268 const int kInputFps = 24;
3269 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3270 stats.input_frame_rate = kInputFps;
3271 stats_proxy_->SetMockStats(stats);
3272
Åsa Persson45b176f2019-09-30 11:19:05 +02003273 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003274 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003275
3276 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003277 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3278 // QP samples.
3279 EXPECT_FALSE(
3280 video_stream_encoder_
3281 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003282 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003283
3284 video_stream_encoder_->Stop();
3285}
3286
3287TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3288 test::ScopedFieldTrials field_trials(
3289 "WebRTC-Video-BalancedDegradationSettings/"
3290 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3291 SetupTest();
3292
3293 // Force input frame rate.
3294 const int kInputFps = 25;
3295 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3296 stats.input_frame_rate = kInputFps;
3297 stats_proxy_->SetMockStats(stats);
3298
Åsa Persson45b176f2019-09-30 11:19:05 +02003299 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003300 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003301
3302 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003303 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3304 // samples.
3305 EXPECT_TRUE(
3306 video_stream_encoder_
3307 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003308 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003309
3310 video_stream_encoder_->Stop();
3311}
3312
3313TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3314 test::ScopedFieldTrials field_trials(
3315 "WebRTC-Video-BalancedDegradationSettings/"
3316 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3317 SetupTest();
3318
3319 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3320
Åsa Persson45b176f2019-09-30 11:19:05 +02003321 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003323
3324 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3325 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003326 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003327
3328 video_stream_encoder_->Stop();
3329}
3330
Åsa Perssonccfb3402019-09-25 15:13:04 +02003331TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003332 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003333 "WebRTC-Video-BalancedDegradationSettings/"
3334 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003335 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003336
Åsa Persson1b247f12019-08-14 17:26:39 +02003337 const int kMinBitrateBps = 425000;
3338 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003339 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003340
Åsa Persson45b176f2019-09-30 11:19:05 +02003341 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003342 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003343 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3344
3345 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3346 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003347 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003348 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003349 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3350
3351 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3352 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003353 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003354 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003355 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3356
Åsa Persson30ab0152019-08-27 12:22:33 +02003357 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3358 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003359 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003360 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003361 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003362 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
3364 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003365 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003366 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003367 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003368
Åsa Persson30ab0152019-08-27 12:22:33 +02003369 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003370 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003371 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003372 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003373 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003374 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3375
3376 video_stream_encoder_->Stop();
3377}
3378
Åsa Perssonccfb3402019-09-25 15:13:04 +02003379TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003380 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3381 test::ScopedFieldTrials field_trials(
3382 "WebRTC-Video-BalancedDegradationSettings/"
3383 "pixels:57600|129600|230400,fps:7|24|24/");
3384 SetupTest();
3385 OnBitrateUpdated(kLowTargetBitrateBps);
3386
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003387 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003388
3389 // Insert frame, expect scaled down:
3390 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3391 InsertFrame();
3392 EXPECT_FALSE(WaitForFrame(1000));
3393 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3394 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3395
3396 // Insert frame, expect scaled down:
3397 // resolution (320x180@24fps).
3398 InsertFrame();
3399 EXPECT_FALSE(WaitForFrame(1000));
3400 EXPECT_LT(source_.sink_wants().max_pixel_count,
3401 source_.last_wants().max_pixel_count);
3402 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3403
3404 // Frame should not be dropped (min pixels per frame reached).
3405 InsertFrameAndWaitForEncoded();
3406
3407 video_stream_encoder_->Stop();
3408}
3409
3410TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003411 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003412 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003413 "WebRTC-Video-BalancedDegradationSettings/"
3414 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003415 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003416
Åsa Persson30ab0152019-08-27 12:22:33 +02003417 const int kResolutionMinBitrateBps = 435000;
3418 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003419 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003420
Åsa Persson45b176f2019-09-30 11:19:05 +02003421 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003422 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003423 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
3425 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3426 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003427 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003428 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003429 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430
3431 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3432 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003433 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003434 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003435 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
3437 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3438 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003439 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003440 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003441 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3442
Åsa Persson30ab0152019-08-27 12:22:33 +02003443 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3444 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003445 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003446 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003447 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3448
3449 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3450 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003451 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003452 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3453
3454 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003455 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003456 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003457 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003458 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003459 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 video_stream_encoder_->Stop();
3462}
3463
Åsa Perssonccfb3402019-09-25 15:13:04 +02003464TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003465 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003466 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003467 "WebRTC-Video-BalancedDegradationSettings/"
3468 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003469 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003470
Åsa Persson30ab0152019-08-27 12:22:33 +02003471 const int kMinBitrateBps = 425000;
3472 const int kTooLowMinBitrateBps = 424000;
3473 const int kResolutionMinBitrateBps = 435000;
3474 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003475 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003476
Åsa Persson45b176f2019-09-30 11:19:05 +02003477 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003478 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003479 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3480
3481 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3482 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003483 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003484 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003485 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3486
3487 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3488 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003489 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003490 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003491 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492
3493 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3494 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003495 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003496 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003497 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3498
3499 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3500 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003501 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003502 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3503
3504 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003505 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003506 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003507 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003508 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003509 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3510
3511 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003512 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003513 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003514 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003515 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3516
3517 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003518 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003519 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003520 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003521 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003522 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3523
Åsa Persson1b247f12019-08-14 17:26:39 +02003524 video_stream_encoder_->Stop();
3525}
3526
mflodmancc3d4422017-08-03 08:27:51 -07003527TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003528 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3529 const int kWidth = 1280;
3530 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003531 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003532 DataRate::BitsPerSec(kTargetBitrateBps),
3533 DataRate::BitsPerSec(kTargetBitrateBps),
3534 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003535
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003536 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003537 AdaptingFrameForwarder source;
3538 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003539 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003540 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003541
Åsa Persson8c1bf952018-09-13 10:42:19 +02003542 int64_t timestamp_ms = kFrameIntervalMs;
3543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003544 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003545 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003546 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3548 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3549 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3550
3551 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003552 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003553 timestamp_ms += kFrameIntervalMs;
3554 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3555 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003556 EXPECT_THAT(source.sink_wants(),
3557 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003558 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3559 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3560 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3561 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3562
3563 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003564 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003565 timestamp_ms += kFrameIntervalMs;
3566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3567 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003568 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003569 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3571 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3572 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3573
Jonathan Yubc771b72017-12-08 17:04:29 -08003574 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003575 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003579 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003580 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003582 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003583 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3584
Jonathan Yubc771b72017-12-08 17:04:29 -08003585 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003586 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003587 timestamp_ms += kFrameIntervalMs;
3588 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3589 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003590 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003591 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003592 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3594 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3595 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3596
Jonathan Yubc771b72017-12-08 17:04:29 -08003597 // Trigger quality adapt down, expect no change (min resolution reached).
3598 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003599 timestamp_ms += kFrameIntervalMs;
3600 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3601 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003602 EXPECT_THAT(source.sink_wants(), FpsMax());
3603 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003604 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3606 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3607 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3608
3609 // Trigger cpu adapt up, expect upscaled resolution (480x270).
Henrik Boström91aa7322020-04-28 12:24:33 +02003610 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003611 timestamp_ms += kFrameIntervalMs;
3612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3613 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003614 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003615 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3617 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3618 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3619
3620 // Trigger cpu adapt up, expect upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003621 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003622 timestamp_ms += kFrameIntervalMs;
3623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3624 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003625 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003626 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3628 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3629 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3630
3631 // Trigger cpu adapt up, expect upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003632 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003633 timestamp_ms += kFrameIntervalMs;
3634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3635 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003636 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003637 last_wants = source.sink_wants();
3638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3639 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003640 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003641 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3642
3643 // Trigger cpu adapt up, no cpu downgrades, expect no change (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003644 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003645 timestamp_ms += kFrameIntervalMs;
3646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3647 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003648 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003649 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3650 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003651 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003652 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3653
3654 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003655 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003656 timestamp_ms += kFrameIntervalMs;
3657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003658 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003659 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003660 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003661 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003663 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003664 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003665
mflodmancc3d4422017-08-03 08:27:51 -07003666 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003667}
3668
mflodmancc3d4422017-08-03 08:27:51 -07003669TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003670 const int kWidth = 640;
3671 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003672
Henrik Boström381d1092020-05-12 18:49:07 +02003673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003674 DataRate::BitsPerSec(kTargetBitrateBps),
3675 DataRate::BitsPerSec(kTargetBitrateBps),
3676 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003677
perkj803d97f2016-11-01 11:45:46 -07003678 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003679 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003680 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003681 }
3682
mflodmancc3d4422017-08-03 08:27:51 -07003683 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003684 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003685 video_source_.IncomingCapturedFrame(CreateFrame(
3686 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003687 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003688 }
3689
mflodmancc3d4422017-08-03 08:27:51 -07003690 video_stream_encoder_->Stop();
3691 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003692 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003693
Ying Wangef3998f2019-12-09 13:06:53 +01003694 EXPECT_METRIC_EQ(
3695 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3696 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003697 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3698}
3699
mflodmancc3d4422017-08-03 08:27:51 -07003700TEST_F(VideoStreamEncoderTest,
3701 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003703 DataRate::BitsPerSec(kTargetBitrateBps),
3704 DataRate::BitsPerSec(kTargetBitrateBps),
3705 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003706 const int kWidth = 640;
3707 const int kHeight = 360;
3708
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003709 video_stream_encoder_->SetSource(&video_source_,
3710 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003711
3712 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3713 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003714 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003715 }
3716
mflodmancc3d4422017-08-03 08:27:51 -07003717 video_stream_encoder_->Stop();
3718 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003719 stats_proxy_.reset();
3720
3721 EXPECT_EQ(0,
3722 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3723}
3724
mflodmancc3d4422017-08-03 08:27:51 -07003725TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003726 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003727 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003728
3729 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003730 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003731 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003732 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3733 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003734
sprang57c2fff2017-01-16 06:24:02 -08003735 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003736 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003738 DataRate::BitsPerSec(kLowTargetBitrateBps),
3739 DataRate::BitsPerSec(kLowTargetBitrateBps),
3740 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003741
sprang57c2fff2017-01-16 06:24:02 -08003742 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003743 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3744 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003745 VideoBitrateAllocation bitrate_allocation =
3746 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003747 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003748 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003749 // TODO(srte): The use of millisecs here looks like an error, but the tests
3750 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003751 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003752
3753 // Not called on second frame.
3754 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3755 .Times(0);
3756 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003757 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3758 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003759 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003760
3761 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003762 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3763 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003764 const int64_t start_time_ms = rtc::TimeMillis();
3765 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3766 video_source_.IncomingCapturedFrame(
3767 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3768 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003769 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003770 }
3771
3772 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003773 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003774
mflodmancc3d4422017-08-03 08:27:51 -07003775 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003776}
3777
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003778TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3779 // 2 TLs configured, temporal layers supported by encoder.
3780 const int kNumTemporalLayers = 2;
3781 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3782 fake_encoder_.SetTemporalLayersSupported(0, true);
3783
3784 // Bitrate allocated across temporal layers.
3785 const int kTl0Bps = kTargetBitrateBps *
3786 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003787 kNumTemporalLayers, /*temporal_id*/ 0,
3788 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003789 const int kTl1Bps = kTargetBitrateBps *
3790 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003791 kNumTemporalLayers, /*temporal_id*/ 1,
3792 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003793 VideoBitrateAllocation expected_bitrate;
3794 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3795 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3796
3797 VerifyAllocatedBitrate(expected_bitrate);
3798 video_stream_encoder_->Stop();
3799}
3800
3801TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3802 // 2 TLs configured, temporal layers not supported by encoder.
3803 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3804 fake_encoder_.SetTemporalLayersSupported(0, false);
3805
3806 // Temporal layers not supported by the encoder.
3807 // Total bitrate should be at ti:0.
3808 VideoBitrateAllocation expected_bitrate;
3809 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3810
3811 VerifyAllocatedBitrate(expected_bitrate);
3812 video_stream_encoder_->Stop();
3813}
3814
3815TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3816 // 2 TLs configured, temporal layers only supported for first stream.
3817 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3818 fake_encoder_.SetTemporalLayersSupported(0, true);
3819 fake_encoder_.SetTemporalLayersSupported(1, false);
3820
3821 const int kS0Bps = 150000;
3822 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003823 kS0Bps *
3824 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3825 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003826 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003827 kS0Bps *
3828 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3829 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003830 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3831 // Temporal layers not supported by si:1.
3832 VideoBitrateAllocation expected_bitrate;
3833 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3834 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3835 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3836
3837 VerifyAllocatedBitrate(expected_bitrate);
3838 video_stream_encoder_->Stop();
3839}
3840
Niels Möller7dc26b72017-12-06 10:27:48 +01003841TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3842 const int kFrameWidth = 1280;
3843 const int kFrameHeight = 720;
3844 const int kFramerate = 24;
3845
Henrik Boström381d1092020-05-12 18:49:07 +02003846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003847 DataRate::BitsPerSec(kTargetBitrateBps),
3848 DataRate::BitsPerSec(kTargetBitrateBps),
3849 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003850 test::FrameForwarder source;
3851 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003852 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003853
3854 // Insert a single frame, triggering initial configuration.
3855 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3856 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3857
3858 EXPECT_EQ(
3859 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3860 kDefaultFramerate);
3861
3862 // Trigger reconfigure encoder (without resetting the entire instance).
3863 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003864 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003865 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3866 video_encoder_config.number_of_streams = 1;
3867 video_encoder_config.video_stream_factory =
3868 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3869 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003870 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003871 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3872
3873 // Detector should be updated with fps limit from codec config.
3874 EXPECT_EQ(
3875 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3876 kFramerate);
3877
3878 // Trigger overuse, max framerate should be reduced.
3879 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3880 stats.input_frame_rate = kFramerate;
3881 stats_proxy_->SetMockStats(stats);
3882 video_stream_encoder_->TriggerCpuOveruse();
3883 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3884 int adapted_framerate =
3885 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3886 EXPECT_LT(adapted_framerate, kFramerate);
3887
3888 // Trigger underuse, max framerate should go back to codec configured fps.
3889 // Set extra low fps, to make sure it's actually reset, not just incremented.
3890 stats = stats_proxy_->GetStats();
3891 stats.input_frame_rate = adapted_framerate / 2;
3892 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003893 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003894 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3895 EXPECT_EQ(
3896 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3897 kFramerate);
3898
3899 video_stream_encoder_->Stop();
3900}
3901
3902TEST_F(VideoStreamEncoderTest,
3903 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3904 const int kFrameWidth = 1280;
3905 const int kFrameHeight = 720;
3906 const int kLowFramerate = 15;
3907 const int kHighFramerate = 25;
3908
Henrik Boström381d1092020-05-12 18:49:07 +02003909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003910 DataRate::BitsPerSec(kTargetBitrateBps),
3911 DataRate::BitsPerSec(kTargetBitrateBps),
3912 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003913 test::FrameForwarder source;
3914 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003915 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003916
3917 // Trigger initial configuration.
3918 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003919 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003920 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3921 video_encoder_config.number_of_streams = 1;
3922 video_encoder_config.video_stream_factory =
3923 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3924 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3925 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003926 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003927 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3928
3929 EXPECT_EQ(
3930 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3931 kLowFramerate);
3932
3933 // Trigger overuse, max framerate should be reduced.
3934 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3935 stats.input_frame_rate = kLowFramerate;
3936 stats_proxy_->SetMockStats(stats);
3937 video_stream_encoder_->TriggerCpuOveruse();
3938 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3939 int adapted_framerate =
3940 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3941 EXPECT_LT(adapted_framerate, kLowFramerate);
3942
3943 // Reconfigure the encoder with a new (higher max framerate), max fps should
3944 // still respect the adaptation.
3945 video_encoder_config.video_stream_factory =
3946 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3947 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3948 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003949 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003950 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3951
3952 EXPECT_EQ(
3953 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3954 adapted_framerate);
3955
3956 // Trigger underuse, max framerate should go back to codec configured fps.
3957 stats = stats_proxy_->GetStats();
3958 stats.input_frame_rate = adapted_framerate;
3959 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003960 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003961 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3962 EXPECT_EQ(
3963 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3964 kHighFramerate);
3965
3966 video_stream_encoder_->Stop();
3967}
3968
mflodmancc3d4422017-08-03 08:27:51 -07003969TEST_F(VideoStreamEncoderTest,
3970 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003971 const int kFrameWidth = 1280;
3972 const int kFrameHeight = 720;
3973 const int kFramerate = 24;
3974
Henrik Boström381d1092020-05-12 18:49:07 +02003975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003976 DataRate::BitsPerSec(kTargetBitrateBps),
3977 DataRate::BitsPerSec(kTargetBitrateBps),
3978 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003979 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003980 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003981 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003982
3983 // Trigger initial configuration.
3984 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003985 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003986 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3987 video_encoder_config.number_of_streams = 1;
3988 video_encoder_config.video_stream_factory =
3989 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3990 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003991 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003992 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003993 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003994
Niels Möller7dc26b72017-12-06 10:27:48 +01003995 EXPECT_EQ(
3996 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3997 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003998
3999 // Trigger overuse, max framerate should be reduced.
4000 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4001 stats.input_frame_rate = kFramerate;
4002 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004003 video_stream_encoder_->TriggerCpuOveruse();
4004 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004005 int adapted_framerate =
4006 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004007 EXPECT_LT(adapted_framerate, kFramerate);
4008
4009 // Change degradation preference to not enable framerate scaling. Target
4010 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004011 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004012 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004013 EXPECT_EQ(
4014 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4015 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004016
mflodmancc3d4422017-08-03 08:27:51 -07004017 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004018}
4019
mflodmancc3d4422017-08-03 08:27:51 -07004020TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004021 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004023 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4024 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4025 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004026 const int kWidth = 640;
4027 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004028
asaperssonfab67072017-04-04 05:51:49 -07004029 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004030
4031 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004032 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004033
4034 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004035 EXPECT_TRUE_WAIT(
4036 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004037
sprangc5d62e22017-04-02 23:53:04 -07004038 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004039
asaperssonfab67072017-04-04 05:51:49 -07004040 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004041 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004042 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004043
4044 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004045 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004046
Henrik Boström2671dac2020-05-19 16:29:09 +02004047 EXPECT_TRUE_WAIT(
4048 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004049
mflodmancc3d4422017-08-03 08:27:51 -07004050 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004051}
4052
mflodmancc3d4422017-08-03 08:27:51 -07004053TEST_F(VideoStreamEncoderTest,
4054 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004055 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004057 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4058 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4059 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004060 const int kWidth = 640;
4061 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004062
4063 // We expect the n initial frames to get dropped.
4064 int i;
4065 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004066 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004067 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004068 }
4069 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004070 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004071 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004072
4073 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004074 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004075
mflodmancc3d4422017-08-03 08:27:51 -07004076 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004077}
4078
mflodmancc3d4422017-08-03 08:27:51 -07004079TEST_F(VideoStreamEncoderTest,
4080 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004081 const int kWidth = 640;
4082 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004083 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004084 DataRate::BitsPerSec(kLowTargetBitrateBps),
4085 DataRate::BitsPerSec(kLowTargetBitrateBps),
4086 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004087
4088 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004089 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004090 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004091
asaperssonfab67072017-04-04 05:51:49 -07004092 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004093 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004094 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004095
mflodmancc3d4422017-08-03 08:27:51 -07004096 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004097}
4098
mflodmancc3d4422017-08-03 08:27:51 -07004099TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004100 const int kWidth = 640;
4101 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004102 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004103
4104 VideoEncoderConfig video_encoder_config;
4105 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4106 // Make format different, to force recreation of encoder.
4107 video_encoder_config.video_format.parameters["foo"] = "foo";
4108 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004109 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004111 DataRate::BitsPerSec(kLowTargetBitrateBps),
4112 DataRate::BitsPerSec(kLowTargetBitrateBps),
4113 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004114
kthelgasonb83797b2017-02-14 11:57:25 -08004115 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004116 video_stream_encoder_->SetSource(&video_source_,
4117 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004118
asaperssonfab67072017-04-04 05:51:49 -07004119 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004120 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004121 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004122
mflodmancc3d4422017-08-03 08:27:51 -07004123 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004124 fake_encoder_.SetQualityScaling(true);
4125}
4126
Åsa Persson139f4dc2019-08-02 09:29:58 +02004127TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4128 webrtc::test::ScopedFieldTrials field_trials(
4129 "WebRTC-Video-QualityScalerSettings/"
4130 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4131 // Reset encoder for field trials to take effect.
4132 ConfigureEncoder(video_encoder_config_.Copy());
4133 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4134 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4135 const int kWidth = 640;
4136 const int kHeight = 360;
4137
Henrik Boström381d1092020-05-12 18:49:07 +02004138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004139 DataRate::BitsPerSec(kTargetBitrateBps),
4140 DataRate::BitsPerSec(kTargetBitrateBps),
4141 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004142 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4143 // Frame should not be dropped.
4144 WaitForEncodedFrame(1);
4145
Henrik Boström381d1092020-05-12 18:49:07 +02004146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004147 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4148 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4149 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004150 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4151 // Frame should not be dropped.
4152 WaitForEncodedFrame(2);
4153
Henrik Boström381d1092020-05-12 18:49:07 +02004154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004155 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4156 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4157 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004158 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4159 // Expect to drop this frame, the wait should time out.
4160 ExpectDroppedFrame();
4161
4162 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004163 EXPECT_TRUE_WAIT(
4164 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004165 video_stream_encoder_->Stop();
4166}
4167
Åsa Perssone644a032019-11-08 15:56:00 +01004168TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4169 webrtc::test::ScopedFieldTrials field_trials(
4170 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4171
4172 // Reset encoder for field trials to take effect.
4173 VideoEncoderConfig config = video_encoder_config_.Copy();
4174 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004175 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004176 ConfigureEncoder(std::move(config));
4177 fake_encoder_.SetQp(kQpLow);
4178
4179 // Enable MAINTAIN_FRAMERATE preference.
4180 AdaptingFrameForwarder source;
4181 source.set_adaptation_enabled(true);
4182 video_stream_encoder_->SetSource(&source,
4183 DegradationPreference::MAINTAIN_FRAMERATE);
4184
4185 // Start at low bitrate.
4186 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004187 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4188 DataRate::BitsPerSec(kLowBitrateBps),
4189 DataRate::BitsPerSec(kLowBitrateBps),
4190 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004191
4192 // Expect first frame to be dropped and resolution to be limited.
4193 const int kWidth = 1280;
4194 const int kHeight = 720;
4195 const int64_t kFrameIntervalMs = 100;
4196 int64_t timestamp_ms = kFrameIntervalMs;
4197 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4198 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004199 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4200 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004201
4202 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004203 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4204 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004205
4206 // Insert frames and advance |min_duration_ms|.
4207 for (size_t i = 1; i <= 10; i++) {
4208 timestamp_ms += kFrameIntervalMs;
4209 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4210 WaitForEncodedFrame(timestamp_ms);
4211 }
4212 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4213 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4214
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004215 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004216
4217 // Insert frame should trigger high BW and release quality limitation.
4218 timestamp_ms += kFrameIntervalMs;
4219 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4220 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004221 // The ramp-up code involves the adaptation queue, give it time to execute.
4222 // TODO(hbos): Can we await an appropriate event instead?
4223 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004224 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004225
4226 // Frame should not be adapted.
4227 timestamp_ms += kFrameIntervalMs;
4228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4229 WaitForEncodedFrame(kWidth, kHeight);
4230 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4231
4232 video_stream_encoder_->Stop();
4233}
4234
mflodmancc3d4422017-08-03 08:27:51 -07004235TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004236 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4237 const int kTooSmallWidth = 10;
4238 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004239 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004240 DataRate::BitsPerSec(kTargetBitrateBps),
4241 DataRate::BitsPerSec(kTargetBitrateBps),
4242 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004243
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004244 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004245 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004246 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004247 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004248 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004249 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4250
4251 // Trigger adapt down, too small frame, expect no change.
4252 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004253 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004254 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004255 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004256 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4257 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4258
mflodmancc3d4422017-08-03 08:27:51 -07004259 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004260}
4261
mflodmancc3d4422017-08-03 08:27:51 -07004262TEST_F(VideoStreamEncoderTest,
4263 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004264 const int kTooSmallWidth = 10;
4265 const int kTooSmallHeight = 10;
4266 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004268 DataRate::BitsPerSec(kTargetBitrateBps),
4269 DataRate::BitsPerSec(kTargetBitrateBps),
4270 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004271
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004272 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004273 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004274 video_stream_encoder_->SetSource(&source,
4275 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004276 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004277 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4278 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4279
4280 // Trigger adapt down, expect limited framerate.
4281 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004282 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004283 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004284 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004285 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4287 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4288
4289 // Trigger adapt down, too small frame, expect no change.
4290 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004291 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004292 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004293 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004294 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4295 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4296 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4297
mflodmancc3d4422017-08-03 08:27:51 -07004298 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004299}
4300
mflodmancc3d4422017-08-03 08:27:51 -07004301TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004302 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004304 DataRate::BitsPerSec(kTargetBitrateBps),
4305 DataRate::BitsPerSec(kTargetBitrateBps),
4306 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004307 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004308 const int kFrameWidth = 1280;
4309 const int kFrameHeight = 720;
4310 video_source_.IncomingCapturedFrame(
4311 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004312 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004313 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004314}
4315
sprangb1ca0732017-02-01 08:38:12 -08004316// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004317TEST_F(VideoStreamEncoderTest,
4318 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004320 DataRate::BitsPerSec(kTargetBitrateBps),
4321 DataRate::BitsPerSec(kTargetBitrateBps),
4322 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004323
4324 const int kFrameWidth = 1280;
4325 const int kFrameHeight = 720;
4326 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004327 // requested by
4328 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004329 video_source_.set_adaptation_enabled(true);
4330
4331 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004332 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004333 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004334
4335 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004336 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004337 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004338 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004339 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004340
asaperssonfab67072017-04-04 05:51:49 -07004341 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004342 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004343 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004344 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004345 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004346
mflodmancc3d4422017-08-03 08:27:51 -07004347 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004348}
sprangfe627f32017-03-29 08:24:59 -07004349
mflodmancc3d4422017-08-03 08:27:51 -07004350TEST_F(VideoStreamEncoderTest,
4351 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004352 const int kFrameWidth = 1280;
4353 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004354
Henrik Boström381d1092020-05-12 18:49:07 +02004355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004356 DataRate::BitsPerSec(kTargetBitrateBps),
4357 DataRate::BitsPerSec(kTargetBitrateBps),
4358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004359 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004360 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004361 video_source_.set_adaptation_enabled(true);
4362
sprang4847ae62017-06-27 07:06:52 -07004363 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004364
4365 video_source_.IncomingCapturedFrame(
4366 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004367 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004368
4369 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004370 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004371
4372 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004373 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004374 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004375 video_source_.IncomingCapturedFrame(
4376 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004377 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004378 }
4379
4380 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004381 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004382 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004383 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004384 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004385 video_source_.IncomingCapturedFrame(
4386 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004387 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004388 ++num_frames_dropped;
4389 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004390 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004391 }
4392 }
4393
sprang4847ae62017-06-27 07:06:52 -07004394 // Add some slack to account for frames dropped by the frame dropper.
4395 const int kErrorMargin = 1;
4396 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004397 kErrorMargin);
4398
4399 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004400 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004401 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004402 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004403 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004404 video_source_.IncomingCapturedFrame(
4405 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004406 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004407 ++num_frames_dropped;
4408 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004409 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004410 }
4411 }
sprang4847ae62017-06-27 07:06:52 -07004412 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004413 kErrorMargin);
4414
4415 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004416 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004417 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004418 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004419 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004420 video_source_.IncomingCapturedFrame(
4421 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004422 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004423 ++num_frames_dropped;
4424 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004425 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004426 }
4427 }
sprang4847ae62017-06-27 07:06:52 -07004428 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004429 kErrorMargin);
4430
4431 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004432 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004433 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004434 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004435 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004436 video_source_.IncomingCapturedFrame(
4437 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004438 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004439 ++num_frames_dropped;
4440 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004441 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004442 }
4443 }
4444 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4445
mflodmancc3d4422017-08-03 08:27:51 -07004446 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004447}
4448
mflodmancc3d4422017-08-03 08:27:51 -07004449TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004450 const int kFramerateFps = 5;
4451 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004452 const int kFrameWidth = 1280;
4453 const int kFrameHeight = 720;
4454
sprang4847ae62017-06-27 07:06:52 -07004455 // Reconfigure encoder with two temporal layers and screensharing, which will
4456 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004457 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004458
Henrik Boström381d1092020-05-12 18:49:07 +02004459 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004460 DataRate::BitsPerSec(kTargetBitrateBps),
4461 DataRate::BitsPerSec(kTargetBitrateBps),
4462 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004463 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004464 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004465 video_source_.set_adaptation_enabled(true);
4466
sprang4847ae62017-06-27 07:06:52 -07004467 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004468
4469 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004470 rtc::VideoSinkWants last_wants;
4471 do {
4472 last_wants = video_source_.sink_wants();
4473
sprangc5d62e22017-04-02 23:53:04 -07004474 // Insert frames to get a new fps estimate...
4475 for (int j = 0; j < kFramerateFps; ++j) {
4476 video_source_.IncomingCapturedFrame(
4477 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004478 if (video_source_.last_sent_width()) {
4479 sink_.WaitForEncodedFrame(timestamp_ms);
4480 }
sprangc5d62e22017-04-02 23:53:04 -07004481 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004482 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004483 }
4484 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004485 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004486 } while (video_source_.sink_wants().max_framerate_fps <
4487 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004488
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004489 EXPECT_THAT(video_source_.sink_wants(),
4490 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004491
mflodmancc3d4422017-08-03 08:27:51 -07004492 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004493}
asaperssonf7e294d2017-06-13 23:25:22 -07004494
mflodmancc3d4422017-08-03 08:27:51 -07004495TEST_F(VideoStreamEncoderTest,
4496 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004497 const int kWidth = 1280;
4498 const int kHeight = 720;
4499 const int64_t kFrameIntervalMs = 150;
4500 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004502 DataRate::BitsPerSec(kTargetBitrateBps),
4503 DataRate::BitsPerSec(kTargetBitrateBps),
4504 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004505
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004506 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004507 AdaptingFrameForwarder source;
4508 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004509 video_stream_encoder_->SetSource(&source,
4510 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004511 timestamp_ms += kFrameIntervalMs;
4512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004513 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004514 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004515 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4516 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4517 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4518
4519 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004520 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004521 timestamp_ms += kFrameIntervalMs;
4522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004523 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004524 EXPECT_THAT(source.sink_wants(),
4525 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4528 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4529
4530 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004531 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004532 timestamp_ms += kFrameIntervalMs;
4533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004534 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004535 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4538 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4539
4540 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004542 timestamp_ms += kFrameIntervalMs;
4543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004544 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004545 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4548 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4549
4550 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004551 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004552 timestamp_ms += kFrameIntervalMs;
4553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004555 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4558 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4559
4560 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004562 timestamp_ms += kFrameIntervalMs;
4563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004565 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4568 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4569
4570 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004571 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004572 timestamp_ms += kFrameIntervalMs;
4573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004574 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004575 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4578 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4579
4580 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004581 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004582 timestamp_ms += kFrameIntervalMs;
4583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004584 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004585 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004586 rtc::VideoSinkWants last_wants = source.sink_wants();
4587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4589 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4590
4591 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004592 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004593 timestamp_ms += kFrameIntervalMs;
4594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004596 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4599 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4600
4601 // Trigger adapt down, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004602 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004603 timestamp_ms += kFrameIntervalMs;
4604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004605 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004606 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4609 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4610
4611 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004612 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004613 timestamp_ms += kFrameIntervalMs;
4614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004616 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4619 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4620
4621 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004622 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004623 timestamp_ms += kFrameIntervalMs;
4624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004625 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004626 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4629 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4630
4631 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004632 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004633 timestamp_ms += kFrameIntervalMs;
4634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004635 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004636 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4638 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4639 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4640
4641 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004642 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004643 timestamp_ms += kFrameIntervalMs;
4644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004645 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004646 EXPECT_THAT(source.sink_wants(), FpsMax());
4647 EXPECT_EQ(source.sink_wants().max_pixel_count,
4648 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004649 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4651 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4652
4653 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004654 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004655 timestamp_ms += kFrameIntervalMs;
4656 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004657 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004658 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004659 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4661 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4662
Åsa Persson30ab0152019-08-27 12:22:33 +02004663 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004664 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004665 timestamp_ms += kFrameIntervalMs;
4666 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004667 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004668 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004669 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4672 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4673
4674 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004675 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004676 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004677 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4678
mflodmancc3d4422017-08-03 08:27:51 -07004679 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004680}
4681
mflodmancc3d4422017-08-03 08:27:51 -07004682TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004683 const int kWidth = 1280;
4684 const int kHeight = 720;
4685 const int64_t kFrameIntervalMs = 150;
4686 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004688 DataRate::BitsPerSec(kTargetBitrateBps),
4689 DataRate::BitsPerSec(kTargetBitrateBps),
4690 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004691
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004692 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004693 AdaptingFrameForwarder source;
4694 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004695 video_stream_encoder_->SetSource(&source,
4696 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004697 timestamp_ms += kFrameIntervalMs;
4698 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004699 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004700 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004701 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4702 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4703 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4704 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4705 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4706 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4707
4708 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004709 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004710 timestamp_ms += kFrameIntervalMs;
4711 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004712 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004713 EXPECT_THAT(source.sink_wants(),
4714 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4717 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4719 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4721
4722 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004723 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004724 timestamp_ms += kFrameIntervalMs;
4725 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004726 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004727 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004728 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4729 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4730 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4732 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4733 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4734
4735 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004736 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004737 timestamp_ms += kFrameIntervalMs;
4738 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004739 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004740 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004741 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4742 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4743 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4745 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4746 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4747
4748 // Trigger cpu adapt up, expect increased fps (640x360@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004749 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004750 timestamp_ms += kFrameIntervalMs;
4751 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004752 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004753 EXPECT_THAT(source.sink_wants(), FpsMax());
4754 EXPECT_EQ(source.sink_wants().max_pixel_count,
4755 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004756 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4757 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4758 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4759 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4760 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4761 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4762
4763 // Trigger quality adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004764 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004765 timestamp_ms += kFrameIntervalMs;
4766 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004767 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004768 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004769 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4770 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4771 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4772 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4773 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4774 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4775
4776 // Trigger cpu adapt up, expect no restriction (1280x720fps@30fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004777 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004778 timestamp_ms += kFrameIntervalMs;
4779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004780 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004781 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004782 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4784 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4786 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4787 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4788 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4789
4790 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004791 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004792 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004793 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4794 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4795
mflodmancc3d4422017-08-03 08:27:51 -07004796 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004797}
4798
mflodmancc3d4422017-08-03 08:27:51 -07004799TEST_F(VideoStreamEncoderTest,
4800 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004801 const int kWidth = 640;
4802 const int kHeight = 360;
4803 const int kFpsLimit = 15;
4804 const int64_t kFrameIntervalMs = 150;
4805 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004807 DataRate::BitsPerSec(kTargetBitrateBps),
4808 DataRate::BitsPerSec(kTargetBitrateBps),
4809 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004810
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004811 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004812 AdaptingFrameForwarder source;
4813 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004814 video_stream_encoder_->SetSource(&source,
4815 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004816 timestamp_ms += kFrameIntervalMs;
4817 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004818 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004819 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004820 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4822 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4823 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4824 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4825 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4826
4827 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004828 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004829 timestamp_ms += kFrameIntervalMs;
4830 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004831 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004832 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004833 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4834 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4835 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4836 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4837 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4838 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4839
4840 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004841 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004842 timestamp_ms += kFrameIntervalMs;
4843 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004844 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004845 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004846 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4847 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4848 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4849 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4850 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4851 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4852
4853 // Trigger cpu adapt up, expect upscaled resolution (640x360@15fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02004854 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004855 timestamp_ms += kFrameIntervalMs;
4856 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004857 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004858 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004859 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4860 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4861 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4862 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4863 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4864 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4865
4866 // Trigger quality adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004867 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004868 timestamp_ms += kFrameIntervalMs;
4869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004870 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004871 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004872 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4873 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4874 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4875 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4876 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4877 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4878
4879 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004880 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004881 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004882 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4883 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4884
mflodmancc3d4422017-08-03 08:27:51 -07004885 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004886}
4887
mflodmancc3d4422017-08-03 08:27:51 -07004888TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004889 const int kFrameWidth = 1920;
4890 const int kFrameHeight = 1080;
4891 // 3/4 of 1920.
4892 const int kAdaptedFrameWidth = 1440;
4893 // 3/4 of 1080 rounded down to multiple of 4.
4894 const int kAdaptedFrameHeight = 808;
4895 const int kFramerate = 24;
4896
Henrik Boström381d1092020-05-12 18:49:07 +02004897 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004898 DataRate::BitsPerSec(kTargetBitrateBps),
4899 DataRate::BitsPerSec(kTargetBitrateBps),
4900 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004901 // Trigger reconfigure encoder (without resetting the entire instance).
4902 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004903 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004904 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4905 video_encoder_config.number_of_streams = 1;
4906 video_encoder_config.video_stream_factory =
4907 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004908 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004909 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004910 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004911
4912 video_source_.set_adaptation_enabled(true);
4913
4914 video_source_.IncomingCapturedFrame(
4915 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004916 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004917
4918 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004919 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004920 video_source_.IncomingCapturedFrame(
4921 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004922 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004923
mflodmancc3d4422017-08-03 08:27:51 -07004924 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004925}
4926
mflodmancc3d4422017-08-03 08:27:51 -07004927TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004928 const int kFrameWidth = 1280;
4929 const int kFrameHeight = 720;
4930 const int kLowFps = 2;
4931 const int kHighFps = 30;
4932
Henrik Boström381d1092020-05-12 18:49:07 +02004933 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004934 DataRate::BitsPerSec(kTargetBitrateBps),
4935 DataRate::BitsPerSec(kTargetBitrateBps),
4936 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004937
4938 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4939 max_framerate_ = kLowFps;
4940
4941 // Insert 2 seconds of 2fps video.
4942 for (int i = 0; i < kLowFps * 2; ++i) {
4943 video_source_.IncomingCapturedFrame(
4944 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4945 WaitForEncodedFrame(timestamp_ms);
4946 timestamp_ms += 1000 / kLowFps;
4947 }
4948
4949 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004950 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004951 DataRate::BitsPerSec(kTargetBitrateBps),
4952 DataRate::BitsPerSec(kTargetBitrateBps),
4953 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004954 video_source_.IncomingCapturedFrame(
4955 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4956 WaitForEncodedFrame(timestamp_ms);
4957 timestamp_ms += 1000 / kLowFps;
4958
4959 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4960
4961 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004962 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004963 const int kFrameIntervalMs = 1000 / kHighFps;
4964 max_framerate_ = kHighFps;
4965 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4966 video_source_.IncomingCapturedFrame(
4967 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4968 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4969 // be dropped if the encoder hans't been updated with the new higher target
4970 // framerate yet, causing it to overshoot the target bitrate and then
4971 // suffering the wrath of the media optimizer.
4972 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4973 timestamp_ms += kFrameIntervalMs;
4974 }
4975
4976 // Don expect correct measurement just yet, but it should be higher than
4977 // before.
4978 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4979
mflodmancc3d4422017-08-03 08:27:51 -07004980 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004981}
4982
mflodmancc3d4422017-08-03 08:27:51 -07004983TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004984 const int kFrameWidth = 1280;
4985 const int kFrameHeight = 720;
4986 const int kTargetBitrateBps = 1000000;
4987
4988 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02004989 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02004990 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004991 DataRate::BitsPerSec(kTargetBitrateBps),
4992 DataRate::BitsPerSec(kTargetBitrateBps),
4993 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004994 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07004995
4996 // Insert a first video frame, causes another bitrate update.
4997 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4998 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
4999 video_source_.IncomingCapturedFrame(
5000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5001 WaitForEncodedFrame(timestamp_ms);
5002
5003 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005004 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5005 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5006 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005007
5008 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005009 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005010 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005011
5012 // Bitrate observer should not be called.
5013 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5014 video_source_.IncomingCapturedFrame(
5015 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5016 ExpectDroppedFrame();
5017
mflodmancc3d4422017-08-03 08:27:51 -07005018 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005019}
ilnik6b826ef2017-06-16 06:53:48 -07005020
Niels Möller4db138e2018-04-19 09:04:13 +02005021TEST_F(VideoStreamEncoderTest,
5022 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5023 const int kFrameWidth = 1280;
5024 const int kFrameHeight = 720;
5025 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005027 DataRate::BitsPerSec(kTargetBitrateBps),
5028 DataRate::BitsPerSec(kTargetBitrateBps),
5029 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005030 video_source_.IncomingCapturedFrame(
5031 CreateFrame(1, kFrameWidth, kFrameHeight));
5032 WaitForEncodedFrame(1);
5033 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5034 .low_encode_usage_threshold_percent,
5035 default_options.low_encode_usage_threshold_percent);
5036 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5037 .high_encode_usage_threshold_percent,
5038 default_options.high_encode_usage_threshold_percent);
5039 video_stream_encoder_->Stop();
5040}
5041
5042TEST_F(VideoStreamEncoderTest,
5043 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5044 const int kFrameWidth = 1280;
5045 const int kFrameHeight = 720;
5046 CpuOveruseOptions hardware_options;
5047 hardware_options.low_encode_usage_threshold_percent = 150;
5048 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005049 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005050
Henrik Boström381d1092020-05-12 18:49:07 +02005051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005052 DataRate::BitsPerSec(kTargetBitrateBps),
5053 DataRate::BitsPerSec(kTargetBitrateBps),
5054 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005055 video_source_.IncomingCapturedFrame(
5056 CreateFrame(1, kFrameWidth, kFrameHeight));
5057 WaitForEncodedFrame(1);
5058 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5059 .low_encode_usage_threshold_percent,
5060 hardware_options.low_encode_usage_threshold_percent);
5061 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5062 .high_encode_usage_threshold_percent,
5063 hardware_options.high_encode_usage_threshold_percent);
5064 video_stream_encoder_->Stop();
5065}
5066
Niels Möller6bb5ab92019-01-11 11:11:10 +01005067TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5068 const int kFrameWidth = 320;
5069 const int kFrameHeight = 240;
5070 const int kFps = 30;
5071 const int kTargetBitrateBps = 120000;
5072 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5073
Henrik Boström381d1092020-05-12 18:49:07 +02005074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005075 DataRate::BitsPerSec(kTargetBitrateBps),
5076 DataRate::BitsPerSec(kTargetBitrateBps),
5077 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005078
5079 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5080 max_framerate_ = kFps;
5081
5082 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5083 fake_encoder_.SimulateOvershoot(1.0);
5084 int num_dropped = 0;
5085 for (int i = 0; i < kNumFramesInRun; ++i) {
5086 video_source_.IncomingCapturedFrame(
5087 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5088 // Wait up to two frame durations for a frame to arrive.
5089 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5090 ++num_dropped;
5091 }
5092 timestamp_ms += 1000 / kFps;
5093 }
5094
Erik Språnga8d48ab2019-02-08 14:17:40 +01005095 // Framerate should be measured to be near the expected target rate.
5096 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5097
5098 // Frame drops should be within 5% of expected 0%.
5099 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005100
5101 // Make encoder produce frames at double the expected bitrate during 3 seconds
5102 // of video, verify number of drops. Rate needs to be slightly changed in
5103 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005104 double overshoot_factor = 2.0;
5105 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5106 // With bitrate adjuster, when need to overshoot even more to trigger
5107 // frame dropping.
5108 overshoot_factor *= 2;
5109 }
5110 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005111 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005112 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5113 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5114 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005115 num_dropped = 0;
5116 for (int i = 0; i < kNumFramesInRun; ++i) {
5117 video_source_.IncomingCapturedFrame(
5118 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5119 // Wait up to two frame durations for a frame to arrive.
5120 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5121 ++num_dropped;
5122 }
5123 timestamp_ms += 1000 / kFps;
5124 }
5125
Henrik Boström381d1092020-05-12 18:49:07 +02005126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005127 DataRate::BitsPerSec(kTargetBitrateBps),
5128 DataRate::BitsPerSec(kTargetBitrateBps),
5129 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005130
5131 // Target framerate should be still be near the expected target, despite
5132 // the frame drops.
5133 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5134
5135 // Frame drops should be within 5% of expected 50%.
5136 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005137
5138 video_stream_encoder_->Stop();
5139}
5140
5141TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5142 const int kFrameWidth = 320;
5143 const int kFrameHeight = 240;
5144 const int kActualInputFps = 24;
5145 const int kTargetBitrateBps = 120000;
5146
5147 ASSERT_GT(max_framerate_, kActualInputFps);
5148
5149 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5150 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005151 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005152 DataRate::BitsPerSec(kTargetBitrateBps),
5153 DataRate::BitsPerSec(kTargetBitrateBps),
5154 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005155
5156 // Insert 3 seconds of video, with an input fps lower than configured max.
5157 for (int i = 0; i < kActualInputFps * 3; ++i) {
5158 video_source_.IncomingCapturedFrame(
5159 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5160 // Wait up to two frame durations for a frame to arrive.
5161 WaitForEncodedFrame(timestamp_ms);
5162 timestamp_ms += 1000 / kActualInputFps;
5163 }
5164
5165 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5166
5167 video_stream_encoder_->Stop();
5168}
5169
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005170TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5171 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005172 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005173 DataRate::BitsPerSec(kTargetBitrateBps),
5174 DataRate::BitsPerSec(kTargetBitrateBps),
5175 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005176
5177 fake_encoder_.BlockNextEncode();
5178 video_source_.IncomingCapturedFrame(
5179 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5180 WaitForEncodedFrame(1);
5181 // On the very first frame full update should be forced.
5182 rect = fake_encoder_.GetLastUpdateRect();
5183 EXPECT_EQ(rect.offset_x, 0);
5184 EXPECT_EQ(rect.offset_y, 0);
5185 EXPECT_EQ(rect.height, codec_height_);
5186 EXPECT_EQ(rect.width, codec_width_);
5187 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5188 // call to ContinueEncode.
5189 video_source_.IncomingCapturedFrame(
5190 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5191 ExpectDroppedFrame();
5192 video_source_.IncomingCapturedFrame(
5193 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5194 ExpectDroppedFrame();
5195 fake_encoder_.ContinueEncode();
5196 WaitForEncodedFrame(3);
5197 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5198 rect = fake_encoder_.GetLastUpdateRect();
5199 EXPECT_EQ(rect.offset_x, 1);
5200 EXPECT_EQ(rect.offset_y, 0);
5201 EXPECT_EQ(rect.width, 10);
5202 EXPECT_EQ(rect.height, 1);
5203
5204 video_source_.IncomingCapturedFrame(
5205 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5206 WaitForEncodedFrame(4);
5207 // Previous frame was encoded, so no accumulation should happen.
5208 rect = fake_encoder_.GetLastUpdateRect();
5209 EXPECT_EQ(rect.offset_x, 0);
5210 EXPECT_EQ(rect.offset_y, 0);
5211 EXPECT_EQ(rect.width, 1);
5212 EXPECT_EQ(rect.height, 1);
5213
5214 video_stream_encoder_->Stop();
5215}
5216
Erik Språngd7329ca2019-02-21 21:19:53 +01005217TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005219 DataRate::BitsPerSec(kTargetBitrateBps),
5220 DataRate::BitsPerSec(kTargetBitrateBps),
5221 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005222
5223 // First frame is always keyframe.
5224 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5225 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005226 EXPECT_THAT(
5227 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005228 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005229
5230 // Insert delta frame.
5231 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5232 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005233 EXPECT_THAT(
5234 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005235 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005236
5237 // Request next frame be a key-frame.
5238 video_stream_encoder_->SendKeyFrame();
5239 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5240 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005241 EXPECT_THAT(
5242 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005243 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005244
5245 video_stream_encoder_->Stop();
5246}
5247
5248TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5249 // Setup simulcast with three streams.
5250 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005252 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5253 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5254 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005255 // Wait for all three layers before triggering event.
5256 sink_.SetNumExpectedLayers(3);
5257
5258 // First frame is always keyframe.
5259 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5260 WaitForEncodedFrame(1);
5261 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005262 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5263 VideoFrameType::kVideoFrameKey,
5264 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005265
5266 // Insert delta frame.
5267 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5268 WaitForEncodedFrame(2);
5269 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005270 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5271 VideoFrameType::kVideoFrameDelta,
5272 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005273
5274 // Request next frame be a key-frame.
5275 // Only first stream is configured to produce key-frame.
5276 video_stream_encoder_->SendKeyFrame();
5277 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5278 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005279
5280 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5281 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005282 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005283 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005284 VideoFrameType::kVideoFrameKey,
5285 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005286
5287 video_stream_encoder_->Stop();
5288}
5289
5290TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5291 // Configure internal source factory and setup test again.
5292 encoder_factory_.SetHasInternalSource(true);
5293 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005294 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005295 DataRate::BitsPerSec(kTargetBitrateBps),
5296 DataRate::BitsPerSec(kTargetBitrateBps),
5297 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005298
5299 // Call encoder directly, simulating internal source where encoded frame
5300 // callback in VideoStreamEncoder is called despite no OnFrame().
5301 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5302 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005303 EXPECT_THAT(
5304 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005305 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005306
Niels Möller8f7ce222019-03-21 15:43:58 +01005307 const std::vector<VideoFrameType> kDeltaFrame = {
5308 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005309 // Need to set timestamp manually since manually for injected frame.
5310 VideoFrame frame = CreateFrame(101, nullptr);
5311 frame.set_timestamp(101);
5312 fake_encoder_.InjectFrame(frame, false);
5313 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005314 EXPECT_THAT(
5315 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005316 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005317
5318 // Request key-frame. The forces a dummy frame down into the encoder.
5319 fake_encoder_.ExpectNullFrame();
5320 video_stream_encoder_->SendKeyFrame();
5321 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005322 EXPECT_THAT(
5323 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005324 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005325
5326 video_stream_encoder_->Stop();
5327}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005328
5329TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5330 // Configure internal source factory and setup test again.
5331 encoder_factory_.SetHasInternalSource(true);
5332 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005334 DataRate::BitsPerSec(kTargetBitrateBps),
5335 DataRate::BitsPerSec(kTargetBitrateBps),
5336 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005337
5338 int64_t timestamp = 1;
5339 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005340 image.SetEncodedData(
5341 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005342 image.capture_time_ms_ = ++timestamp;
5343 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5344 const int64_t kEncodeFinishDelayMs = 10;
5345 image.timing_.encode_start_ms = timestamp;
5346 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5347 fake_encoder_.InjectEncodedImage(image);
5348 // Wait for frame without incrementing clock.
5349 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5350 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5351 // capture timestamp should be kEncodeFinishDelayMs in the past.
5352 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5353 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5354 kEncodeFinishDelayMs);
5355
5356 video_stream_encoder_->Stop();
5357}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005358
5359TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5360 // Configure internal source factory and setup test again.
5361 encoder_factory_.SetHasInternalSource(true);
5362 ResetEncoder("H264", 1, 1, 1, false);
5363
5364 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5365 image._frameType = VideoFrameType::kVideoFrameKey;
5366
5367 CodecSpecificInfo codec_specific_info;
5368 codec_specific_info.codecType = kVideoCodecH264;
5369
5370 RTPFragmentationHeader fragmentation;
5371 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5372 fragmentation.fragmentationOffset[0] = 4;
5373 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5374
5375 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5376 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5377
5378 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5379 testing::ElementsAreArray(optimal_sps));
5380 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5381 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5382 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5383 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5384
5385 video_stream_encoder_->Stop();
5386}
5387
5388TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5389 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5390 0x00, 0x00, 0x03, 0x03, 0xF4,
5391 0x05, 0x03, 0xC7, 0xC0};
5392
5393 // Configure internal source factory and setup test again.
5394 encoder_factory_.SetHasInternalSource(true);
5395 ResetEncoder("H264", 1, 1, 1, false);
5396
5397 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5398 image._frameType = VideoFrameType::kVideoFrameKey;
5399
5400 CodecSpecificInfo codec_specific_info;
5401 codec_specific_info.codecType = kVideoCodecH264;
5402
5403 RTPFragmentationHeader fragmentation;
5404 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5405 fragmentation.fragmentationOffset[0] = 4;
5406 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5407
5408 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5409 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5410
5411 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5412 testing::ElementsAreArray(optimal_sps));
5413 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5414 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5415 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5416 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5417
5418 video_stream_encoder_->Stop();
5419}
5420
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005421TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5422 const int kFrameWidth = 1280;
5423 const int kFrameHeight = 720;
5424 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5425
Henrik Boström381d1092020-05-12 18:49:07 +02005426 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005427 DataRate::BitsPerSec(kTargetBitrateBps),
5428 DataRate::BitsPerSec(kTargetBitrateBps),
5429 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5431
5432 // Insert a first video frame. It should be dropped because of downscale in
5433 // resolution.
5434 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5435 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5436 frame.set_rotation(kVideoRotation_270);
5437 video_source_.IncomingCapturedFrame(frame);
5438
5439 ExpectDroppedFrame();
5440
5441 // Second frame is downscaled.
5442 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5443 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5444 frame.set_rotation(kVideoRotation_90);
5445 video_source_.IncomingCapturedFrame(frame);
5446
5447 WaitForEncodedFrame(timestamp_ms);
5448 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5449
5450 // Insert another frame, also downscaled.
5451 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5452 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5453 frame.set_rotation(kVideoRotation_180);
5454 video_source_.IncomingCapturedFrame(frame);
5455
5456 WaitForEncodedFrame(timestamp_ms);
5457 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5458
5459 video_stream_encoder_->Stop();
5460}
5461
Erik Språng5056af02019-09-02 15:53:11 +02005462TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5463 const int kFrameWidth = 320;
5464 const int kFrameHeight = 180;
5465
5466 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005468 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5469 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5470 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005471 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005472 /*rtt_ms=*/0,
5473 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005474
5475 // Insert a first video frame so that encoder gets configured.
5476 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5477 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5478 frame.set_rotation(kVideoRotation_270);
5479 video_source_.IncomingCapturedFrame(frame);
5480 WaitForEncodedFrame(timestamp_ms);
5481
5482 // Set a target rate below the minimum allowed by the codec settings.
5483 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005484 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5485 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005487 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005488 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005489 /*link_allocation=*/target_rate,
5490 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005491 /*rtt_ms=*/0,
5492 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005493 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5494
5495 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5496 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5497 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005498 DataRate allocation_sum =
5499 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005500 EXPECT_EQ(min_rate, allocation_sum);
5501 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5502
5503 video_stream_encoder_->Stop();
5504}
5505
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005506TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005508 DataRate::BitsPerSec(kTargetBitrateBps),
5509 DataRate::BitsPerSec(kTargetBitrateBps),
5510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005511 // Capture a frame and wait for it to synchronize with the encoder thread.
5512 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5513 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5514 WaitForEncodedFrame(1);
5515
5516 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5517 ASSERT_TRUE(prev_rate_settings.has_value());
5518 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5519 kDefaultFramerate);
5520
5521 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5522 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5523 timestamp_ms += 1000 / kDefaultFramerate;
5524 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5525 WaitForEncodedFrame(timestamp_ms);
5526 }
5527 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5528 kDefaultFramerate);
5529 // Capture larger frame to trigger a reconfigure.
5530 codec_height_ *= 2;
5531 codec_width_ *= 2;
5532 timestamp_ms += 1000 / kDefaultFramerate;
5533 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5534 WaitForEncodedFrame(timestamp_ms);
5535
5536 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5537 auto current_rate_settings =
5538 fake_encoder_.GetAndResetLastRateControlSettings();
5539 // Ensure we have actually reconfigured twice
5540 // The rate settings should have been set again even though
5541 // they haven't changed.
5542 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005543 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005544
5545 video_stream_encoder_->Stop();
5546}
5547
philipeld9cc8c02019-09-16 14:53:40 +02005548struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005549 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5550 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5551 MOCK_METHOD(void,
5552 RequestEncoderSwitch,
5553 (const webrtc::SdpVideoFormat& format),
5554 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005555};
5556
5557TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5558 constexpr int kDontCare = 100;
5559
5560 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5561 video_send_config_.encoder_settings.encoder_switch_request_callback =
5562 &switch_callback;
5563 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5564 encoder_config.codec_type = kVideoCodecVP8;
5565 webrtc::test::ScopedFieldTrials field_trial(
5566 "WebRTC-NetworkCondition-EncoderSwitch/"
5567 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5568 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5569
5570 // Reset encoder for new configuration to take effect.
5571 ConfigureEncoder(std::move(encoder_config));
5572
5573 // Send one frame to trigger ReconfigureEncoder.
5574 video_source_.IncomingCapturedFrame(
5575 CreateFrame(kDontCare, kDontCare, kDontCare));
5576
5577 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005578 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5579 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005580 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005581 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005582
Henrik Boström381d1092020-05-12 18:49:07 +02005583 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005584 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5585 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5586 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005587 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005588 /*rtt_ms=*/0,
5589 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005590
5591 video_stream_encoder_->Stop();
5592}
5593
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005594TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5595 constexpr int kDontCare = 100;
5596
5597 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5598 video_send_config_.encoder_settings.encoder_switch_request_callback =
5599 &switch_callback;
5600 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5601 encoder_config.codec_type = kVideoCodecVP8;
5602 webrtc::test::ScopedFieldTrials field_trial(
5603 "WebRTC-NetworkCondition-EncoderSwitch/"
5604 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5605 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5606
5607 // Reset encoder for new configuration to take effect.
5608 ConfigureEncoder(std::move(encoder_config));
5609
5610 // Send one frame to trigger ReconfigureEncoder.
5611 video_source_.IncomingCapturedFrame(
5612 CreateFrame(kDontCare, kDontCare, kDontCare));
5613
5614 using Config = EncoderSwitchRequestCallback::Config;
5615 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5616 .Times(0);
5617
Henrik Boström381d1092020-05-12 18:49:07 +02005618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005619 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5620 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5621 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5622 /*fraction_lost=*/0,
5623 /*rtt_ms=*/0,
5624 /*cwnd_reduce_ratio=*/0);
5625
5626 video_stream_encoder_->Stop();
5627}
5628
philipeld9cc8c02019-09-16 14:53:40 +02005629TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5630 constexpr int kSufficientBitrateToNotDrop = 1000;
5631 constexpr int kHighRes = 500;
5632 constexpr int kLowRes = 100;
5633
5634 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5635 video_send_config_.encoder_settings.encoder_switch_request_callback =
5636 &switch_callback;
5637 webrtc::test::ScopedFieldTrials field_trial(
5638 "WebRTC-NetworkCondition-EncoderSwitch/"
5639 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5640 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5641 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5642 encoder_config.codec_type = kVideoCodecH264;
5643
5644 // Reset encoder for new configuration to take effect.
5645 ConfigureEncoder(std::move(encoder_config));
5646
5647 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5648 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5649 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005650 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005651 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5652 /*stable_target_bitrate=*/
5653 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5654 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005655 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005656 /*rtt_ms=*/0,
5657 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005658
5659 // Send one frame to trigger ReconfigureEncoder.
5660 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5661 WaitForEncodedFrame(1);
5662
5663 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005664 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5665 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005666 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005667 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005668
5669 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5670 WaitForEncodedFrame(2);
5671
5672 video_stream_encoder_->Stop();
5673}
5674
philipel9b058032020-02-10 11:30:00 +01005675TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5676 constexpr int kDontCare = 100;
5677 StrictMock<MockEncoderSelector> encoder_selector;
5678 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5679 &fake_encoder_, &encoder_selector);
5680 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5681
5682 // Reset encoder for new configuration to take effect.
5683 ConfigureEncoder(video_encoder_config_.Copy());
5684
5685 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5686
5687 video_source_.IncomingCapturedFrame(
5688 CreateFrame(kDontCare, kDontCare, kDontCare));
5689 video_stream_encoder_->Stop();
5690
5691 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5692 // to it's factory, so in order for the encoder instance in the
5693 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5694 // reset the |video_stream_encoder_| here.
5695 video_stream_encoder_.reset();
5696}
5697
5698TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5699 constexpr int kDontCare = 100;
5700
5701 NiceMock<MockEncoderSelector> encoder_selector;
5702 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5703 video_send_config_.encoder_settings.encoder_switch_request_callback =
5704 &switch_callback;
5705 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5706 &fake_encoder_, &encoder_selector);
5707 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5708
5709 // Reset encoder for new configuration to take effect.
5710 ConfigureEncoder(video_encoder_config_.Copy());
5711
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005712 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005713 .WillByDefault(Return(SdpVideoFormat("AV1")));
5714 EXPECT_CALL(switch_callback,
5715 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5716 Field(&SdpVideoFormat::name, "AV1"))));
5717
Henrik Boström381d1092020-05-12 18:49:07 +02005718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005719 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5720 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5721 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005722 /*fraction_lost=*/0,
5723 /*rtt_ms=*/0,
5724 /*cwnd_reduce_ratio=*/0);
5725
5726 video_stream_encoder_->Stop();
5727}
5728
5729TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5730 constexpr int kSufficientBitrateToNotDrop = 1000;
5731 constexpr int kDontCare = 100;
5732
5733 NiceMock<MockVideoEncoder> video_encoder;
5734 NiceMock<MockEncoderSelector> encoder_selector;
5735 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5736 video_send_config_.encoder_settings.encoder_switch_request_callback =
5737 &switch_callback;
5738 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5739 &video_encoder, &encoder_selector);
5740 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5741
5742 // Reset encoder for new configuration to take effect.
5743 ConfigureEncoder(video_encoder_config_.Copy());
5744
5745 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5746 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5747 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005749 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5750 /*stable_target_bitrate=*/
5751 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5752 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005753 /*fraction_lost=*/0,
5754 /*rtt_ms=*/0,
5755 /*cwnd_reduce_ratio=*/0);
5756
5757 ON_CALL(video_encoder, Encode(_, _))
5758 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5759 ON_CALL(encoder_selector, OnEncoderBroken())
5760 .WillByDefault(Return(SdpVideoFormat("AV2")));
5761
5762 rtc::Event encode_attempted;
5763 EXPECT_CALL(switch_callback,
5764 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5765 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5766 EXPECT_EQ(format.name, "AV2");
5767 encode_attempted.Set();
5768 });
5769
5770 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5771 encode_attempted.Wait(3000);
5772
5773 video_stream_encoder_->Stop();
5774
5775 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5776 // to it's factory, so in order for the encoder instance in the
5777 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5778 // reset the |video_stream_encoder_| here.
5779 video_stream_encoder_.reset();
5780}
5781
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005782TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005783 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005784 const int kFrameWidth = 320;
5785 const int kFrameHeight = 180;
5786
5787 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005788 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005790 /*target_bitrate=*/rate,
5791 /*stable_target_bitrate=*/rate,
5792 /*link_allocation=*/rate,
5793 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005794 /*rtt_ms=*/0,
5795 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005796
5797 // Insert a first video frame so that encoder gets configured.
5798 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5799 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5800 frame.set_rotation(kVideoRotation_270);
5801 video_source_.IncomingCapturedFrame(frame);
5802 WaitForEncodedFrame(timestamp_ms);
5803 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5804
5805 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005806 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005808 /*target_bitrate=*/new_stable_rate,
5809 /*stable_target_bitrate=*/new_stable_rate,
5810 /*link_allocation=*/rate,
5811 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005812 /*rtt_ms=*/0,
5813 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005814 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5815 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5816 video_stream_encoder_->Stop();
5817}
5818
5819TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005820 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005821 const int kFrameWidth = 320;
5822 const int kFrameHeight = 180;
5823
5824 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005825 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005827 /*target_bitrate=*/rate,
5828 /*stable_target_bitrate=*/rate,
5829 /*link_allocation=*/rate,
5830 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005831 /*rtt_ms=*/0,
5832 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005833
5834 // Insert a first video frame so that encoder gets configured.
5835 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5836 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5837 frame.set_rotation(kVideoRotation_270);
5838 video_source_.IncomingCapturedFrame(frame);
5839 WaitForEncodedFrame(timestamp_ms);
5840 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5841
5842 // Set a higher target rate without changing the link_allocation. Should not
5843 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005844 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005845 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005846 /*target_bitrate=*/rate,
5847 /*stable_target_bitrate=*/new_stable_rate,
5848 /*link_allocation=*/rate,
5849 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005850 /*rtt_ms=*/0,
5851 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005852 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5853 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5854 video_stream_encoder_->Stop();
5855}
5856
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005857TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5858 test::ScopedFieldTrials field_trials(
5859 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5860 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5861 const int kFramerateFps = 30;
5862 const int kWidth = 1920;
5863 const int kHeight = 1080;
5864 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5865 // Works on screenshare mode.
5866 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5867 // We rely on the automatic resolution adaptation, but we handle framerate
5868 // adaptation manually by mocking the stats proxy.
5869 video_source_.set_adaptation_enabled(true);
5870
5871 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005873 DataRate::BitsPerSec(kTargetBitrateBps),
5874 DataRate::BitsPerSec(kTargetBitrateBps),
5875 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005876 video_stream_encoder_->SetSource(&video_source_,
5877 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005878 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005879
5880 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5881 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5882
5883 // Pass enough frames with the full update to trigger animation detection.
5884 for (int i = 0; i < kNumFrames; ++i) {
5885 int64_t timestamp_ms =
5886 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5887 frame.set_ntp_time_ms(timestamp_ms);
5888 frame.set_timestamp_us(timestamp_ms * 1000);
5889 video_source_.IncomingCapturedFrame(frame);
5890 WaitForEncodedFrame(timestamp_ms);
5891 }
5892
5893 // Resolution should be limited.
5894 rtc::VideoSinkWants expected;
5895 expected.max_framerate_fps = kFramerateFps;
5896 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005897 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005898
5899 // Pass one frame with no known update.
5900 // Resolution cap should be removed immediately.
5901 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5902 frame.set_ntp_time_ms(timestamp_ms);
5903 frame.set_timestamp_us(timestamp_ms * 1000);
5904 frame.clear_update_rect();
5905
5906 video_source_.IncomingCapturedFrame(frame);
5907 WaitForEncodedFrame(timestamp_ms);
5908
5909 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005910 EXPECT_THAT(video_source_.sink_wants(),
5911 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005912
5913 video_stream_encoder_->Stop();
5914}
5915
perkj26091b12016-09-01 01:17:40 -07005916} // namespace webrtc