blob: 46104569c82bfe0840b3b764e5de7767ccd2dd33 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020029#include "call/adaptation/test/fake_adaptation_constraint.h"
30#include "call/adaptation/test/fake_adaptation_listener.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020035#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020036#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010037#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020039#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020042#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010043#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020044#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020045#include "system_wrappers/include/sleep.h"
46#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020052#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020053#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070054
55namespace webrtc {
56
sprang57c2fff2017-01-16 06:24:02 -080057using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020058using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020059using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020060using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Ge;
62using ::testing::Gt;
63using ::testing::Le;
64using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010065using ::testing::Matcher;
66using ::testing::NiceMock;
67using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020068using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080069
perkj803d97f2016-11-01 11:45:46 -070070namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020071const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010072const int kQpLow = 1;
73const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinFramerateFps = 2;
75const int kMinBalancedFramerateFps = 7;
76const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080077const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010078const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020079const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010080const uint32_t kSimulcastTargetBitrateBps = 3150000;
81const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080082const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070083const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020084const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020085const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020086const VideoEncoder::ResolutionBitrateLimits
87 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
88const VideoEncoder::ResolutionBitrateLimits
89 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080090
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020091uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
92 0x00, 0x00, 0x03, 0x03, 0xF4,
93 0x05, 0x03, 0xC7, 0xE0, 0x1B,
94 0x41, 0x10, 0x8D, 0x00};
95
perkj803d97f2016-11-01 11:45:46 -070096class TestBuffer : public webrtc::I420Buffer {
97 public:
98 TestBuffer(rtc::Event* event, int width, int height)
99 : I420Buffer(width, height), event_(event) {}
100
101 private:
102 friend class rtc::RefCountedObject<TestBuffer>;
103 ~TestBuffer() override {
104 if (event_)
105 event_->Set();
106 }
107 rtc::Event* const event_;
108};
109
Noah Richards51db4212019-06-12 06:59:12 -0700110// A fake native buffer that can't be converted to I420.
111class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
112 public:
113 FakeNativeBuffer(rtc::Event* event, int width, int height)
114 : event_(event), width_(width), height_(height) {}
115 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
116 int width() const override { return width_; }
117 int height() const override { return height_; }
118 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
119 return nullptr;
120 }
121
122 private:
123 friend class rtc::RefCountedObject<FakeNativeBuffer>;
124 ~FakeNativeBuffer() override {
125 if (event_)
126 event_->Set();
127 }
128 rtc::Event* const event_;
129 const int width_;
130 const int height_;
131};
132
Niels Möller7dc26b72017-12-06 10:27:48 +0100133class CpuOveruseDetectorProxy : public OveruseFrameDetector {
134 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200135 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
136 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200137 last_target_framerate_fps_(-1),
138 framerate_updated_event_(true /* manual_reset */,
139 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100140 virtual ~CpuOveruseDetectorProxy() {}
141
142 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200143 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 last_target_framerate_fps_ = framerate_fps;
145 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200146 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100147 }
148
149 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200150 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100151 return last_target_framerate_fps_;
152 }
153
Niels Möller4db138e2018-04-19 09:04:13 +0200154 CpuOveruseOptions GetOptions() { return options_; }
155
Henrik Boström381d1092020-05-12 18:49:07 +0200156 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
157
Niels Möller7dc26b72017-12-06 10:27:48 +0100158 private:
Markus Handella3765182020-07-08 13:13:32 +0200159 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100160 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200161 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100162};
163
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200164class FakeVideoSourceRestrictionsListener
165 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200166 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200167 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200168 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200169 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200170 RTC_DCHECK(was_restrictions_updated_);
171 }
172
173 rtc::Event* restrictions_updated_event() {
174 return &restrictions_updated_event_;
175 }
176
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200177 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200178 void OnVideoSourceRestrictionsUpdated(
179 VideoSourceRestrictions restrictions,
180 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200181 rtc::scoped_refptr<Resource> reason,
182 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200183 was_restrictions_updated_ = true;
184 restrictions_updated_event_.Set();
185 }
186
187 private:
188 bool was_restrictions_updated_;
189 rtc::Event restrictions_updated_event_;
190};
191
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200192auto WantsFps(Matcher<int> fps_matcher) {
193 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
194 fps_matcher);
195}
196
197auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
198 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
199 AllOf(max_pixel_matcher, Gt(0)));
200}
201
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200202auto ResolutionMax() {
203 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200204 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200205 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
206 Eq(absl::nullopt)));
207}
208
209auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200210 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200211}
212
213auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200214 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200215}
216
217auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200218 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200219}
220
221auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200222 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200223}
224
225auto FpsMaxResolutionMax() {
226 return AllOf(FpsMax(), ResolutionMax());
227}
228
229auto UnlimitedSinkWants() {
230 return AllOf(FpsUnlimited(), ResolutionMax());
231}
232
233auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
234 Matcher<int> fps_range_matcher;
235
236 if (last_frame_pixels <= 320 * 240) {
237 fps_range_matcher = AllOf(Ge(7), Le(10));
238 } else if (last_frame_pixels <= 480 * 270) {
239 fps_range_matcher = AllOf(Ge(10), Le(15));
240 } else if (last_frame_pixels <= 640 * 480) {
241 fps_range_matcher = Ge(15);
242 } else {
243 fps_range_matcher = Eq(kDefaultFramerate);
244 }
245 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
246 fps_range_matcher);
247}
248
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200249auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
250 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
251 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
252}
253
254auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
255 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
256}
257
258auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
259 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
260}
261
262auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
263 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
264 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
265}
266
267auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
268 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
269 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
270}
271
272auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
273 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
274 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
275}
276
277auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
278 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
279 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
280}
281
mflodmancc3d4422017-08-03 08:27:51 -0700282class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700283 public:
Niels Möller213618e2018-07-24 09:29:58 +0200284 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200285 const VideoStreamEncoderSettings& settings,
286 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100287 : VideoStreamEncoder(Clock::GetRealTimeClock(),
288 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200289 stats_proxy,
290 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200291 std::unique_ptr<OveruseFrameDetector>(
292 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100293 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100294 task_queue_factory),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200295 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200296 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200297 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200298 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200299 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200300 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200301 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100302 }
perkj803d97f2016-11-01 11:45:46 -0700303
Henrik Boström381d1092020-05-12 18:49:07 +0200304 void SetSourceAndWaitForRestrictionsUpdated(
305 rtc::VideoSourceInterface<VideoFrame>* source,
306 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200307 FakeVideoSourceRestrictionsListener listener;
308 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200309 SetSource(source, degradation_preference);
310 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200311 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200312 }
313
314 void SetSourceAndWaitForFramerateUpdated(
315 rtc::VideoSourceInterface<VideoFrame>* source,
316 const DegradationPreference& degradation_preference) {
317 overuse_detector_proxy_->framerate_updated_event()->Reset();
318 SetSource(source, degradation_preference);
319 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
320 }
321
322 void OnBitrateUpdatedAndWaitForManagedResources(
323 DataRate target_bitrate,
324 DataRate stable_target_bitrate,
325 DataRate link_allocation,
326 uint8_t fraction_lost,
327 int64_t round_trip_time_ms,
328 double cwnd_reduce_ratio) {
329 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
330 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
331 // Bitrate is updated on the encoder queue.
332 WaitUntilTaskQueueIsIdle();
333 // Give the managed resources time to react to the new bitrate.
334 // TODO(hbos): Can we await an appropriate event instead?
335 WaitUntilAdaptationTaskQueueIsIdle();
336 }
337
338 void WaitUntilAdaptationTaskQueueIsIdle() {
339 rtc::Event event;
340 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
341 ASSERT_TRUE(event.Wait(5000));
342 }
343
kthelgason2fc52542017-03-03 00:24:41 -0800344 // This is used as a synchronisation mechanism, to make sure that the
345 // encoder queue is not blocked before we start sending it frames.
346 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100347 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200348 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800349 ASSERT_TRUE(event.Wait(5000));
350 }
351
Henrik Boström91aa7322020-04-28 12:24:33 +0200352 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200353 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200354 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200355 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200356 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200357 event.Set();
358 });
359 ASSERT_TRUE(event.Wait(5000));
360 }
361 void TriggerCpuUnderuse() {
362 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200363 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200364 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200365 event.Set();
366 });
367 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200368 }
kthelgason876222f2016-11-29 01:44:11 -0800369
Henrik Boström91aa7322020-04-28 12:24:33 +0200370 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200371 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200372 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200373 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200374 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200375 event.Set();
376 });
377 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200378 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200379 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200380 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200381 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200382 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200383 event.Set();
384 });
385 ASSERT_TRUE(event.Wait(5000));
386 }
387
Niels Möller7dc26b72017-12-06 10:27:48 +0100388 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200389 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
390 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200391 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700392};
393
asapersson5f7226f2016-11-25 04:37:00 -0800394class VideoStreamFactory
395 : public VideoEncoderConfig::VideoStreamFactoryInterface {
396 public:
sprangfda496a2017-06-15 04:21:07 -0700397 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
398 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800399 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700400 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800401 }
402
403 private:
404 std::vector<VideoStream> CreateEncoderStreams(
405 int width,
406 int height,
407 const VideoEncoderConfig& encoder_config) override {
408 std::vector<VideoStream> streams =
409 test::CreateVideoStreams(width, height, encoder_config);
410 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100411 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700412 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800413 }
414 return streams;
415 }
sprangfda496a2017-06-15 04:21:07 -0700416
asapersson5f7226f2016-11-25 04:37:00 -0800417 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700418 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800419};
420
Noah Richards51db4212019-06-12 06:59:12 -0700421// Simulates simulcast behavior and makes highest stream resolutions divisible
422// by 4.
423class CroppingVideoStreamFactory
424 : public VideoEncoderConfig::VideoStreamFactoryInterface {
425 public:
426 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
427 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
428 EXPECT_GT(num_temporal_layers, 0u);
429 EXPECT_GT(framerate, 0);
430 }
431
432 private:
433 std::vector<VideoStream> CreateEncoderStreams(
434 int width,
435 int height,
436 const VideoEncoderConfig& encoder_config) override {
437 std::vector<VideoStream> streams = test::CreateVideoStreams(
438 width - width % 4, height - height % 4, encoder_config);
439 for (VideoStream& stream : streams) {
440 stream.num_temporal_layers = num_temporal_layers_;
441 stream.max_framerate = framerate_;
442 }
443 return streams;
444 }
445
446 const size_t num_temporal_layers_;
447 const int framerate_;
448};
449
sprangb1ca0732017-02-01 08:38:12 -0800450class AdaptingFrameForwarder : public test::FrameForwarder {
451 public:
452 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700453 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800454
455 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200456 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800457 adaptation_enabled_ = enabled;
458 }
459
asaperssonfab67072017-04-04 05:51:49 -0700460 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200461 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800462 return adaptation_enabled_;
463 }
464
asapersson09f05612017-05-15 23:40:18 -0700465 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200466 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700467 return last_wants_;
468 }
469
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200470 absl::optional<int> last_sent_width() const { return last_width_; }
471 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800472
sprangb1ca0732017-02-01 08:38:12 -0800473 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
474 int cropped_width = 0;
475 int cropped_height = 0;
476 int out_width = 0;
477 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700478 if (adaption_enabled()) {
479 if (adapter_.AdaptFrameResolution(
480 video_frame.width(), video_frame.height(),
481 video_frame.timestamp_us() * 1000, &cropped_width,
482 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100483 VideoFrame adapted_frame =
484 VideoFrame::Builder()
485 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
486 nullptr, out_width, out_height))
487 .set_timestamp_rtp(99)
488 .set_timestamp_ms(99)
489 .set_rotation(kVideoRotation_0)
490 .build();
sprangc5d62e22017-04-02 23:53:04 -0700491 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100492 if (video_frame.has_update_rect()) {
493 adapted_frame.set_update_rect(
494 video_frame.update_rect().ScaleWithFrame(
495 video_frame.width(), video_frame.height(), 0, 0,
496 video_frame.width(), video_frame.height(), out_width,
497 out_height));
498 }
sprangc5d62e22017-04-02 23:53:04 -0700499 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800500 last_width_.emplace(adapted_frame.width());
501 last_height_.emplace(adapted_frame.height());
502 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200503 last_width_ = absl::nullopt;
504 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700505 }
sprangb1ca0732017-02-01 08:38:12 -0800506 } else {
507 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800508 last_width_.emplace(video_frame.width());
509 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800510 }
511 }
512
513 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
514 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200515 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200516 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100517 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200518 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800519 }
sprangb1ca0732017-02-01 08:38:12 -0800520 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200521 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
522 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200523 absl::optional<int> last_width_;
524 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800525};
sprangc5d62e22017-04-02 23:53:04 -0700526
Niels Möller213618e2018-07-24 09:29:58 +0200527// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700528class MockableSendStatisticsProxy : public SendStatisticsProxy {
529 public:
530 MockableSendStatisticsProxy(Clock* clock,
531 const VideoSendStream::Config& config,
532 VideoEncoderConfig::ContentType content_type)
533 : SendStatisticsProxy(clock, config, content_type) {}
534
535 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200536 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700537 if (mock_stats_)
538 return *mock_stats_;
539 return SendStatisticsProxy::GetStats();
540 }
541
Niels Möller213618e2018-07-24 09:29:58 +0200542 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200543 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200544 if (mock_stats_)
545 return mock_stats_->input_frame_rate;
546 return SendStatisticsProxy::GetInputFrameRate();
547 }
sprangc5d62e22017-04-02 23:53:04 -0700548 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200549 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700550 mock_stats_.emplace(stats);
551 }
552
553 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200554 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700555 mock_stats_.reset();
556 }
557
558 private:
Markus Handella3765182020-07-08 13:13:32 +0200559 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200560 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700561};
562
sprang4847ae62017-06-27 07:06:52 -0700563class MockBitrateObserver : public VideoBitrateAllocationObserver {
564 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200565 MOCK_METHOD(void,
566 OnBitrateAllocationUpdated,
567 (const VideoBitrateAllocation&),
568 (override));
sprang4847ae62017-06-27 07:06:52 -0700569};
570
philipel9b058032020-02-10 11:30:00 +0100571class MockEncoderSelector
572 : public VideoEncoderFactory::EncoderSelectorInterface {
573 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200574 MOCK_METHOD(void,
575 OnCurrentEncoder,
576 (const SdpVideoFormat& format),
577 (override));
578 MOCK_METHOD(absl::optional<SdpVideoFormat>,
579 OnAvailableBitrate,
580 (const DataRate& rate),
581 (override));
582 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100583};
584
perkj803d97f2016-11-01 11:45:46 -0700585} // namespace
586
mflodmancc3d4422017-08-03 08:27:51 -0700587class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700588 public:
589 static const int kDefaultTimeoutMs = 30 * 1000;
590
mflodmancc3d4422017-08-03 08:27:51 -0700591 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700592 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700593 codec_width_(320),
594 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200595 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200596 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700597 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200598 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700599 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700600 Clock::GetRealTimeClock(),
601 video_send_config_,
602 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700603 sink_(&fake_encoder_) {}
604
605 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700606 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700607 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200608 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800609 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200610 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200611 video_send_config_.rtp.payload_name = "FAKE";
612 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700613
Per512ecb32016-09-23 15:52:06 +0200614 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200615 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700616 video_encoder_config.video_stream_factory =
617 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100618 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700619
620 // Framerate limit is specified by the VideoStreamFactory.
621 std::vector<VideoStream> streams =
622 video_encoder_config.video_stream_factory->CreateEncoderStreams(
623 codec_width_, codec_height_, video_encoder_config);
624 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100625 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700626
Niels Möllerf1338562018-04-26 09:51:47 +0200627 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800628 }
629
Niels Möllerf1338562018-04-26 09:51:47 +0200630 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700631 if (video_stream_encoder_)
632 video_stream_encoder_->Stop();
633 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200634 stats_proxy_.get(), video_send_config_.encoder_settings,
635 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700636 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
637 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700638 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700639 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
640 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200641 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700642 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800643 }
644
645 void ResetEncoder(const std::string& payload_name,
646 size_t num_streams,
647 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700648 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700649 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200650 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800651
652 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200653 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800654 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100655 video_encoder_config.max_bitrate_bps =
656 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800657 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700658 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
659 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700660 video_encoder_config.content_type =
661 screenshare ? VideoEncoderConfig::ContentType::kScreen
662 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700663 if (payload_name == "VP9") {
664 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
665 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200666 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700667 video_encoder_config.encoder_specific_settings =
668 new rtc::RefCountedObject<
669 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
670 }
Niels Möllerf1338562018-04-26 09:51:47 +0200671 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700672 }
673
sprang57c2fff2017-01-16 06:24:02 -0800674 VideoFrame CreateFrame(int64_t ntp_time_ms,
675 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100676 VideoFrame frame =
677 VideoFrame::Builder()
678 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
679 destruction_event, codec_width_, codec_height_))
680 .set_timestamp_rtp(99)
681 .set_timestamp_ms(99)
682 .set_rotation(kVideoRotation_0)
683 .build();
sprang57c2fff2017-01-16 06:24:02 -0800684 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700685 return frame;
686 }
687
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100688 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
689 rtc::Event* destruction_event,
690 int offset_x) const {
691 VideoFrame frame =
692 VideoFrame::Builder()
693 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
694 destruction_event, codec_width_, codec_height_))
695 .set_timestamp_rtp(99)
696 .set_timestamp_ms(99)
697 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100698 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100699 .build();
700 frame.set_ntp_time_ms(ntp_time_ms);
701 return frame;
702 }
703
sprang57c2fff2017-01-16 06:24:02 -0800704 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100705 VideoFrame frame =
706 VideoFrame::Builder()
707 .set_video_frame_buffer(
708 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
709 .set_timestamp_rtp(99)
710 .set_timestamp_ms(99)
711 .set_rotation(kVideoRotation_0)
712 .build();
sprang57c2fff2017-01-16 06:24:02 -0800713 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700714 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700715 return frame;
716 }
717
Noah Richards51db4212019-06-12 06:59:12 -0700718 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
719 rtc::Event* destruction_event,
720 int width,
721 int height) const {
722 VideoFrame frame =
723 VideoFrame::Builder()
724 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
725 destruction_event, width, height))
726 .set_timestamp_rtp(99)
727 .set_timestamp_ms(99)
728 .set_rotation(kVideoRotation_0)
729 .build();
730 frame.set_ntp_time_ms(ntp_time_ms);
731 return frame;
732 }
733
734 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
735 rtc::Event* destruction_event) const {
736 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
737 codec_height_);
738 }
739
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100740 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
741 MockBitrateObserver bitrate_observer;
742 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
743
744 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
745 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200746 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100747 DataRate::BitsPerSec(kTargetBitrateBps),
748 DataRate::BitsPerSec(kTargetBitrateBps),
749 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100750
751 video_source_.IncomingCapturedFrame(
752 CreateFrame(1, codec_width_, codec_height_));
753 WaitForEncodedFrame(1);
754 }
755
sprang4847ae62017-06-27 07:06:52 -0700756 void WaitForEncodedFrame(int64_t expected_ntp_time) {
757 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100758 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700759 }
760
761 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
762 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100763 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700764 return ok;
765 }
766
767 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
768 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100769 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700770 }
771
772 void ExpectDroppedFrame() {
773 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100774 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700775 }
776
777 bool WaitForFrame(int64_t timeout_ms) {
778 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100779 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700780 return ok;
781 }
782
perkj26091b12016-09-01 01:17:40 -0700783 class TestEncoder : public test::FakeEncoder {
784 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100785 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700786
asaperssonfab67072017-04-04 05:51:49 -0700787 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200788 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700789 return config_;
790 }
791
792 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200793 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700794 block_next_encode_ = true;
795 }
796
Erik Språngaed30702018-11-05 12:57:17 +0100797 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200798 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100799 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100800 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100801 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100802 info.scaling_settings = VideoEncoder::ScalingSettings(
803 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100804 }
805 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100806 for (int i = 0; i < kMaxSpatialLayers; ++i) {
807 if (temporal_layers_supported_[i]) {
808 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
809 info.fps_allocation[i].resize(num_layers);
810 }
811 }
Erik Språngaed30702018-11-05 12:57:17 +0100812 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200813
814 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100815 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100816 return info;
kthelgason876222f2016-11-29 01:44:11 -0800817 }
818
Erik Språngb7cb7b52019-02-26 15:52:33 +0100819 int32_t RegisterEncodeCompleteCallback(
820 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200821 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100822 encoded_image_callback_ = callback;
823 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
824 }
825
perkjfa10b552016-10-02 23:45:26 -0700826 void ContinueEncode() { continue_encode_event_.Set(); }
827
828 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
829 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200830 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700831 EXPECT_EQ(timestamp_, timestamp);
832 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
833 }
834
kthelgason2fc52542017-03-03 00:24:41 -0800835 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200836 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800837 quality_scaling_ = b;
838 }
kthelgasonad9010c2017-02-14 00:46:51 -0800839
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100840 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200841 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100842 requested_resolution_alignment_ = requested_resolution_alignment;
843 }
844
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100845 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200846 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100847 is_hardware_accelerated_ = is_hardware_accelerated;
848 }
849
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100850 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
851 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200852 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100853 temporal_layers_supported_[spatial_idx] = supported;
854 }
855
Sergey Silkin6456e352019-07-08 17:56:40 +0200856 void SetResolutionBitrateLimits(
857 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200858 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200859 resolution_bitrate_limits_ = thresholds;
860 }
861
sprangfe627f32017-03-29 08:24:59 -0700862 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200863 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700864 force_init_encode_failed_ = force_failure;
865 }
866
Niels Möller6bb5ab92019-01-11 11:11:10 +0100867 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200868 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100869 rate_factor_ = rate_factor;
870 }
871
Erik Språngd7329ca2019-02-21 21:19:53 +0100872 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200873 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100874 return last_framerate_;
875 }
876
Erik Språngd7329ca2019-02-21 21:19:53 +0100877 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200878 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100879 return last_update_rect_;
880 }
881
Niels Möller87e2d782019-03-07 10:18:23 +0100882 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200883 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100884 return last_frame_types_;
885 }
886
887 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100888 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100889 keyframe ? VideoFrameType::kVideoFrameKey
890 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100891 {
Markus Handella3765182020-07-08 13:13:32 +0200892 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100893 last_frame_types_ = frame_type;
894 }
Niels Möllerb859b322019-03-07 12:40:01 +0100895 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100896 }
897
Erik Språngb7cb7b52019-02-26 15:52:33 +0100898 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200899 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100900 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
901 }
902
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200903 void SetEncodedImageData(
904 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200906 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200907 }
908
Erik Språngd7329ca2019-02-21 21:19:53 +0100909 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200910 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100911 expect_null_frame_ = true;
912 }
913
Erik Språng5056af02019-09-02 15:53:11 +0200914 absl::optional<VideoEncoder::RateControlParameters>
915 GetAndResetLastRateControlSettings() {
916 auto settings = last_rate_control_settings_;
917 last_rate_control_settings_.reset();
918 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100919 }
920
Sergey Silkin5ee69672019-07-02 14:18:34 +0200921 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200922 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200923 return num_encoder_initializations_;
924 }
925
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200926 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200927 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200928 return num_set_rates_;
929 }
930
perkjfa10b552016-10-02 23:45:26 -0700931 private:
perkj26091b12016-09-01 01:17:40 -0700932 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100933 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700934 bool block_encode;
935 {
Markus Handella3765182020-07-08 13:13:32 +0200936 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100937 if (expect_null_frame_) {
938 EXPECT_EQ(input_image.timestamp(), 0u);
939 EXPECT_EQ(input_image.width(), 1);
940 last_frame_types_ = *frame_types;
941 expect_null_frame_ = false;
942 } else {
943 EXPECT_GT(input_image.timestamp(), timestamp_);
944 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
945 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
946 }
perkj26091b12016-09-01 01:17:40 -0700947
948 timestamp_ = input_image.timestamp();
949 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700950 last_input_width_ = input_image.width();
951 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700952 block_encode = block_next_encode_;
953 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100954 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100955 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700956 }
Niels Möllerb859b322019-03-07 12:40:01 +0100957 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700958 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700959 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700960 return result;
961 }
962
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200963 std::unique_ptr<RTPFragmentationHeader> EncodeHook(
964 EncodedImage* encoded_image,
965 CodecSpecificInfo* codec_specific) override {
966 {
967 MutexLock lock(&mutex_);
968 codec_specific->codecType = config_.codecType;
969 }
970 MutexLock lock(&local_mutex_);
971 if (encoded_image_data_) {
972 encoded_image->SetEncodedData(encoded_image_data_);
973 if (codec_specific->codecType == kVideoCodecH264) {
974 auto fragmentation = std::make_unique<RTPFragmentationHeader>();
975 fragmentation->VerifyAndAllocateFragmentationHeader(1);
976 fragmentation->fragmentationOffset[0] = 4;
977 fragmentation->fragmentationLength[0] = encoded_image->size() - 4;
978 return fragmentation;
979 }
980 }
981 return nullptr;
982 }
983
sprangfe627f32017-03-29 08:24:59 -0700984 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200985 const Settings& settings) override {
986 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200987
Markus Handella3765182020-07-08 13:13:32 +0200988 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100989 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200990
991 ++num_encoder_initializations_;
992
Erik Språng82fad3d2018-03-21 09:57:23 +0100993 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700994 // Simulate setting up temporal layers, in order to validate the life
995 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100996 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200997 frame_buffer_controller_ =
998 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700999 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001000 if (force_init_encode_failed_) {
1001 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001002 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001003 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001004
Erik Språngb7cb7b52019-02-26 15:52:33 +01001005 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001006 return res;
1007 }
1008
Erik Språngb7cb7b52019-02-26 15:52:33 +01001009 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001010 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001011 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1012 initialized_ = EncoderState::kUninitialized;
1013 return FakeEncoder::Release();
1014 }
1015
Erik Språng16cb8f52019-04-12 13:59:09 +02001016 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001017 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001018 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001019 VideoBitrateAllocation adjusted_rate_allocation;
1020 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1021 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001022 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001023 adjusted_rate_allocation.SetBitrate(
1024 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001025 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001026 rate_factor_));
1027 }
1028 }
1029 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001030 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001031 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001032 RateControlParameters adjusted_paramters = parameters;
1033 adjusted_paramters.bitrate = adjusted_rate_allocation;
1034 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001035 }
1036
Markus Handella3765182020-07-08 13:13:32 +02001037 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001038 enum class EncoderState {
1039 kUninitialized,
1040 kInitializationFailed,
1041 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001042 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1043 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001044 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001045 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1046 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1047 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1048 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1049 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1050 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
1051 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001052 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1053 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001054 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001055 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001056 absl::optional<bool>
1057 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001058 local_mutex_);
1059 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1060 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1061 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001062 absl::optional<VideoEncoder::RateControlParameters>
1063 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001064 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1065 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001066 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001067 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001068 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1069 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001070 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001071 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001072 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001073 RTC_GUARDED_BY(local_mutex_);
1074 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001075 };
1076
mflodmancc3d4422017-08-03 08:27:51 -07001077 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001078 public:
1079 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001080 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001081
perkj26091b12016-09-01 01:17:40 -07001082 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001083 EXPECT_TRUE(
1084 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1085 }
1086
1087 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1088 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001089 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001090 if (!encoded_frame_event_.Wait(timeout_ms))
1091 return false;
perkj26091b12016-09-01 01:17:40 -07001092 {
Markus Handella3765182020-07-08 13:13:32 +02001093 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001094 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001095 }
1096 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001097 return true;
perkj26091b12016-09-01 01:17:40 -07001098 }
1099
sprangb1ca0732017-02-01 08:38:12 -08001100 void WaitForEncodedFrame(uint32_t expected_width,
1101 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001102 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001103 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001104 }
1105
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001106 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001107 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001108 uint32_t width = 0;
1109 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001110 {
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001112 width = last_width_;
1113 height = last_height_;
1114 }
1115 EXPECT_EQ(expected_height, height);
1116 EXPECT_EQ(expected_width, width);
1117 }
1118
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001119 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1120 int width = 0;
1121 int height = 0;
1122 {
Markus Handella3765182020-07-08 13:13:32 +02001123 MutexLock lock(&mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001124 width = last_width_;
1125 height = last_height_;
1126 }
1127 EXPECT_EQ(width % resolution_alignment, 0);
1128 EXPECT_EQ(height % resolution_alignment, 0);
1129 }
1130
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001131 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1132 VideoRotation rotation;
1133 {
Markus Handella3765182020-07-08 13:13:32 +02001134 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001135 rotation = last_rotation_;
1136 }
1137 EXPECT_EQ(expected_rotation, rotation);
1138 }
1139
kthelgason2fc52542017-03-03 00:24:41 -08001140 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001141
sprangc5d62e22017-04-02 23:53:04 -07001142 bool WaitForFrame(int64_t timeout_ms) {
1143 return encoded_frame_event_.Wait(timeout_ms);
1144 }
1145
perkj26091b12016-09-01 01:17:40 -07001146 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001147 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001148 expect_frames_ = false;
1149 }
1150
asaperssonfab67072017-04-04 05:51:49 -07001151 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001152 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001153 return number_of_reconfigurations_;
1154 }
1155
asaperssonfab67072017-04-04 05:51:49 -07001156 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001157 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001158 return min_transmit_bitrate_bps_;
1159 }
1160
Erik Språngd7329ca2019-02-21 21:19:53 +01001161 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001162 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001163 num_expected_layers_ = num_layers;
1164 }
1165
Erik Språngb7cb7b52019-02-26 15:52:33 +01001166 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001167 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001168 return last_capture_time_ms_;
1169 }
1170
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001171 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001172 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001173 return std::move(last_encoded_image_data_);
1174 }
1175
perkj26091b12016-09-01 01:17:40 -07001176 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001177 Result OnEncodedImage(
1178 const EncodedImage& encoded_image,
1179 const CodecSpecificInfo* codec_specific_info,
Danil Chapovalov6d008a82020-07-22 19:49:36 +02001180 const RTPFragmentationHeader* /*fragmentation*/) override {
Markus Handella3765182020-07-08 13:13:32 +02001181 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001182 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001183 last_encoded_image_data_ = std::vector<uint8_t>(
1184 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001185 uint32_t timestamp = encoded_image.Timestamp();
1186 if (last_timestamp_ != timestamp) {
1187 num_received_layers_ = 1;
1188 } else {
1189 ++num_received_layers_;
1190 }
1191 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001192 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001193 last_width_ = encoded_image._encodedWidth;
1194 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001195 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001196 if (num_received_layers_ == num_expected_layers_) {
1197 encoded_frame_event_.Set();
1198 }
sprangb1ca0732017-02-01 08:38:12 -08001199 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001200 }
1201
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001202 void OnEncoderConfigurationChanged(
1203 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001204 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001205 VideoEncoderConfig::ContentType content_type,
1206 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001207 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001208 ++number_of_reconfigurations_;
1209 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1210 }
1211
Markus Handella3765182020-07-08 13:13:32 +02001212 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001213 TestEncoder* test_encoder_;
1214 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001215 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001216 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001217 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001218 uint32_t last_height_ = 0;
1219 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001220 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001221 size_t num_expected_layers_ = 1;
1222 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001223 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001224 int number_of_reconfigurations_ = 0;
1225 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001226 };
1227
Sergey Silkin5ee69672019-07-02 14:18:34 +02001228 class VideoBitrateAllocatorProxyFactory
1229 : public VideoBitrateAllocatorFactory {
1230 public:
1231 VideoBitrateAllocatorProxyFactory()
1232 : bitrate_allocator_factory_(
1233 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1234
1235 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1236 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001237 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001238 codec_config_ = codec;
1239 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1240 }
1241
1242 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001243 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001244 return codec_config_;
1245 }
1246
1247 private:
1248 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1249
Markus Handella3765182020-07-08 13:13:32 +02001250 mutable Mutex mutex_;
1251 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001252 };
1253
perkj26091b12016-09-01 01:17:40 -07001254 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001255 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001256 int codec_width_;
1257 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001258 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001259 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001260 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001261 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001262 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001263 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001264 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001265 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001266 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001267 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001268};
1269
mflodmancc3d4422017-08-03 08:27:51 -07001270TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001271 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001272 DataRate::BitsPerSec(kTargetBitrateBps),
1273 DataRate::BitsPerSec(kTargetBitrateBps),
1274 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001275 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001276 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001277 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001278 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001279 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001280}
1281
mflodmancc3d4422017-08-03 08:27:51 -07001282TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001283 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001284 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001285 // The encoder will cache up to one frame for a short duration. Adding two
1286 // frames means that the first frame will be dropped and the second frame will
1287 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001288 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001289 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001290 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001291
Henrik Boström381d1092020-05-12 18:49:07 +02001292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001293 DataRate::BitsPerSec(kTargetBitrateBps),
1294 DataRate::BitsPerSec(kTargetBitrateBps),
1295 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001296
Sebastian Janssona3177052018-04-10 13:05:49 +02001297 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001298 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001299 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1300
1301 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001302 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001303}
1304
mflodmancc3d4422017-08-03 08:27:51 -07001305TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001306 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps),
1309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001310 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001311 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001312
Henrik Boström381d1092020-05-12 18:49:07 +02001313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1314 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1315 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001316 // The encoder will cache up to one frame for a short duration. Adding two
1317 // frames means that the first frame will be dropped and the second frame will
1318 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001319 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001320 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001321
Henrik Boström381d1092020-05-12 18:49:07 +02001322 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001323 DataRate::BitsPerSec(kTargetBitrateBps),
1324 DataRate::BitsPerSec(kTargetBitrateBps),
1325 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001326 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001327 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1328 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001329 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001330}
1331
mflodmancc3d4422017-08-03 08:27:51 -07001332TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001334 DataRate::BitsPerSec(kTargetBitrateBps),
1335 DataRate::BitsPerSec(kTargetBitrateBps),
1336 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001337 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001338 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001339
1340 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001341 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001342
perkja49cbd32016-09-16 07:53:41 -07001343 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001344 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001345 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001346}
1347
mflodmancc3d4422017-08-03 08:27:51 -07001348TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001349 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001350 DataRate::BitsPerSec(kTargetBitrateBps),
1351 DataRate::BitsPerSec(kTargetBitrateBps),
1352 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001353
perkja49cbd32016-09-16 07:53:41 -07001354 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001355 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001356
mflodmancc3d4422017-08-03 08:27:51 -07001357 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001358 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001359 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001360 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1361 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001362}
1363
mflodmancc3d4422017-08-03 08:27:51 -07001364TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001365 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001366 DataRate::BitsPerSec(kTargetBitrateBps),
1367 DataRate::BitsPerSec(kTargetBitrateBps),
1368 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001369
1370 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001371 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001372 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001373 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1374 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001375 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1376 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001377 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001378 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001379
mflodmancc3d4422017-08-03 08:27:51 -07001380 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001381}
1382
Noah Richards51db4212019-06-12 06:59:12 -07001383TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001384 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001385 DataRate::BitsPerSec(kTargetBitrateBps),
1386 DataRate::BitsPerSec(kTargetBitrateBps),
1387 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001388
1389 rtc::Event frame_destroyed_event;
1390 video_source_.IncomingCapturedFrame(
1391 CreateFakeNativeFrame(1, &frame_destroyed_event));
1392 ExpectDroppedFrame();
1393 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1394 video_stream_encoder_->Stop();
1395}
1396
1397TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1398 // Use the cropping factory.
1399 video_encoder_config_.video_stream_factory =
1400 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1401 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1402 kMaxPayloadLength);
1403 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1404
1405 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001407 DataRate::BitsPerSec(kTargetBitrateBps),
1408 DataRate::BitsPerSec(kTargetBitrateBps),
1409 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001410 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1411 WaitForEncodedFrame(1);
1412 // The encoder will have been configured once.
1413 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1414 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1415 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1416
1417 // Now send in a fake frame that needs to be cropped as the width/height
1418 // aren't divisible by 4 (see CreateEncoderStreams above).
1419 rtc::Event frame_destroyed_event;
1420 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1421 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1422 ExpectDroppedFrame();
1423 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1424 video_stream_encoder_->Stop();
1425}
1426
Ying Wang9b881ab2020-02-07 14:29:32 +01001427TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001428 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001429 DataRate::BitsPerSec(kTargetBitrateBps),
1430 DataRate::BitsPerSec(kTargetBitrateBps),
1431 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001432 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1433 WaitForEncodedFrame(1);
1434
Henrik Boström381d1092020-05-12 18:49:07 +02001435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps),
1438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001439 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1440 // frames. Adding two frames means that the first frame will be dropped and
1441 // the second frame will be sent to the encoder.
1442 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1443 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1444 WaitForEncodedFrame(3);
1445 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1446 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1447 WaitForEncodedFrame(5);
1448 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1449 video_stream_encoder_->Stop();
1450}
1451
mflodmancc3d4422017-08-03 08:27:51 -07001452TEST_F(VideoStreamEncoderTest,
1453 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001455 DataRate::BitsPerSec(kTargetBitrateBps),
1456 DataRate::BitsPerSec(kTargetBitrateBps),
1457 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001458 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001459
1460 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001461 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001462 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001463 // The encoder will have been configured once when the first frame is
1464 // received.
1465 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001466
1467 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001468 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001469 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001470 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001471 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001472
1473 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001474 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001475 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001476 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001477 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001478
mflodmancc3d4422017-08-03 08:27:51 -07001479 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001480}
1481
mflodmancc3d4422017-08-03 08:27:51 -07001482TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001484 DataRate::BitsPerSec(kTargetBitrateBps),
1485 DataRate::BitsPerSec(kTargetBitrateBps),
1486 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001487
1488 // Capture a frame and wait for it to synchronize with the encoder thread.
1489 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001490 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001491 // The encoder will have been configured once.
1492 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001493 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1494 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1495
1496 codec_width_ *= 2;
1497 codec_height_ *= 2;
1498 // Capture a frame with a higher resolution and wait for it to synchronize
1499 // with the encoder thread.
1500 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001501 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001502 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1503 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001504 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001505
mflodmancc3d4422017-08-03 08:27:51 -07001506 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001507}
1508
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001509TEST_F(VideoStreamEncoderTest,
1510 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001512 DataRate::BitsPerSec(kTargetBitrateBps),
1513 DataRate::BitsPerSec(kTargetBitrateBps),
1514 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001515
1516 // Capture a frame and wait for it to synchronize with the encoder thread.
1517 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1518 WaitForEncodedFrame(1);
1519
1520 VideoEncoderConfig video_encoder_config;
1521 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1522 // Changing the max payload data length recreates encoder.
1523 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1524 kMaxPayloadLength / 2);
1525
1526 // Capture a frame and wait for it to synchronize with the encoder thread.
1527 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1528 WaitForEncodedFrame(2);
1529 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1530
1531 video_stream_encoder_->Stop();
1532}
1533
Sergey Silkin5ee69672019-07-02 14:18:34 +02001534TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps),
1538 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001539
1540 VideoEncoderConfig video_encoder_config;
1541 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1542 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1543 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1544 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1545 kMaxPayloadLength);
1546
1547 // Capture a frame and wait for it to synchronize with the encoder thread.
1548 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1549 WaitForEncodedFrame(1);
1550 // The encoder will have been configured once when the first frame is
1551 // received.
1552 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1553 EXPECT_EQ(kTargetBitrateBps,
1554 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1555 EXPECT_EQ(kStartBitrateBps,
1556 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1557
Sergey Silkin6456e352019-07-08 17:56:40 +02001558 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1559 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001560 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1561 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1562 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1563 kMaxPayloadLength);
1564
1565 // Capture a frame and wait for it to synchronize with the encoder thread.
1566 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1567 WaitForEncodedFrame(2);
1568 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1569 // Bitrate limits have changed - rate allocator should be reconfigured,
1570 // encoder should not be reconfigured.
1571 EXPECT_EQ(kTargetBitrateBps * 2,
1572 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1573 EXPECT_EQ(kStartBitrateBps * 2,
1574 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1575 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1576
1577 video_stream_encoder_->Stop();
1578}
1579
Sergey Silkin6456e352019-07-08 17:56:40 +02001580TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001581 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001582 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps),
1585 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001586
Sergey Silkincd02eba2020-01-20 14:48:40 +01001587 const uint32_t kMinEncBitrateKbps = 100;
1588 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001589 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001590 /*frame_size_pixels=*/codec_width_ * codec_height_,
1591 /*min_start_bitrate_bps=*/0,
1592 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1593 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001594 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1595
Sergey Silkincd02eba2020-01-20 14:48:40 +01001596 VideoEncoderConfig video_encoder_config;
1597 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1598 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1599 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1600 (kMinEncBitrateKbps + 1) * 1000;
1601 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1602 kMaxPayloadLength);
1603
1604 // When both encoder and app provide bitrate limits, the intersection of
1605 // provided sets should be used.
1606 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1607 WaitForEncodedFrame(1);
1608 EXPECT_EQ(kMaxEncBitrateKbps,
1609 bitrate_allocator_factory_.codec_config().maxBitrate);
1610 EXPECT_EQ(kMinEncBitrateKbps + 1,
1611 bitrate_allocator_factory_.codec_config().minBitrate);
1612
1613 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1614 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1615 (kMinEncBitrateKbps - 1) * 1000;
1616 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1617 kMaxPayloadLength);
1618 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001619 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001620 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001621 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001622 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001623 bitrate_allocator_factory_.codec_config().minBitrate);
1624
Sergey Silkincd02eba2020-01-20 14:48:40 +01001625 video_stream_encoder_->Stop();
1626}
1627
1628TEST_F(VideoStreamEncoderTest,
1629 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001631 DataRate::BitsPerSec(kTargetBitrateBps),
1632 DataRate::BitsPerSec(kTargetBitrateBps),
1633 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001634
1635 const uint32_t kMinAppBitrateKbps = 100;
1636 const uint32_t kMaxAppBitrateKbps = 200;
1637 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1638 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1639 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1640 /*frame_size_pixels=*/codec_width_ * codec_height_,
1641 /*min_start_bitrate_bps=*/0,
1642 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1643 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1644 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1645
1646 VideoEncoderConfig video_encoder_config;
1647 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1648 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1649 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1650 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001651 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1652 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001653
Sergey Silkincd02eba2020-01-20 14:48:40 +01001654 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1655 WaitForEncodedFrame(1);
1656 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001657 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001658 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001659 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001660
1661 video_stream_encoder_->Stop();
1662}
1663
1664TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001665 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001667 DataRate::BitsPerSec(kTargetBitrateBps),
1668 DataRate::BitsPerSec(kTargetBitrateBps),
1669 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001670
1671 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001672 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001673 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001674 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001675 fake_encoder_.SetResolutionBitrateLimits(
1676 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1677
1678 VideoEncoderConfig video_encoder_config;
1679 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1680 video_encoder_config.max_bitrate_bps = 0;
1681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1682 kMaxPayloadLength);
1683
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001684 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001685 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1686 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001687 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1688 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001689 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1690 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1691
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001692 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001693 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1694 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001695 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1696 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001697 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1698 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1699
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001700 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001701 // encoder for 360p should be used.
1702 video_source_.IncomingCapturedFrame(
1703 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1704 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001705 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1706 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001707 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1708 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1709
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001710 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001711 // ignored.
1712 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1713 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001714 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1715 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001716 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1717 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001718 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1719 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001720 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1721 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1722
1723 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1724 // for 270p should be used.
1725 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1726 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001727 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1728 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001729 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1730 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1731
1732 video_stream_encoder_->Stop();
1733}
1734
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001735TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001736 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001737 DataRate::BitsPerSec(kTargetBitrateBps),
1738 DataRate::BitsPerSec(kTargetBitrateBps),
1739 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001740
1741 VideoEncoderConfig video_encoder_config;
1742 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1743 video_encoder_config.max_bitrate_bps = 0;
1744 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1745 kMaxPayloadLength);
1746
1747 // Encode 720p frame to get the default encoder target bitrate.
1748 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1749 WaitForEncodedFrame(1);
1750 const uint32_t kDefaultTargetBitrateFor720pKbps =
1751 bitrate_allocator_factory_.codec_config()
1752 .simulcastStream[0]
1753 .targetBitrate;
1754
1755 // Set the max recommended encoder bitrate to something lower than the default
1756 // target bitrate.
1757 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1758 1280 * 720, 10 * 1000, 10 * 1000,
1759 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1760 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1761
1762 // Change resolution to trigger encoder reinitialization.
1763 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1764 WaitForEncodedFrame(2);
1765 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1766 WaitForEncodedFrame(3);
1767
1768 // Ensure the target bitrate is capped by the max bitrate.
1769 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1770 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1771 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1772 .simulcastStream[0]
1773 .targetBitrate *
1774 1000,
1775 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1776
1777 video_stream_encoder_->Stop();
1778}
1779
mflodmancc3d4422017-08-03 08:27:51 -07001780TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001781 EXPECT_TRUE(video_source_.has_sinks());
1782 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001783 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001784 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001785 EXPECT_FALSE(video_source_.has_sinks());
1786 EXPECT_TRUE(new_video_source.has_sinks());
1787
mflodmancc3d4422017-08-03 08:27:51 -07001788 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001789}
1790
mflodmancc3d4422017-08-03 08:27:51 -07001791TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001792 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001793 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001794 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001795 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001796}
1797
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001798TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1799 constexpr int kRequestedResolutionAlignment = 7;
1800 video_source_.set_adaptation_enabled(true);
1801 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001803 DataRate::BitsPerSec(kTargetBitrateBps),
1804 DataRate::BitsPerSec(kTargetBitrateBps),
1805 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001806
1807 // On the 1st frame, we should have initialized the encoder and
1808 // asked for its resolution requirements.
1809 video_source_.IncomingCapturedFrame(
1810 CreateFrame(1, codec_width_, codec_height_));
1811 WaitForEncodedFrame(1);
1812 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1813 kRequestedResolutionAlignment);
1814
1815 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1816 // (It's up the to the encoder to potentially drop the previous frame,
1817 // to avoid coding back-to-back keyframes.)
1818 video_source_.IncomingCapturedFrame(
1819 CreateFrame(2, codec_width_, codec_height_));
1820 WaitForEncodedFrame(2);
1821 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1822
1823 video_stream_encoder_->Stop();
1824}
1825
Jonathan Yubc771b72017-12-08 17:04:29 -08001826TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1827 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001828 const int kWidth = 1280;
1829 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001830
1831 // We rely on the automatic resolution adaptation, but we handle framerate
1832 // adaptation manually by mocking the stats proxy.
1833 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001834
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001835 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001837 DataRate::BitsPerSec(kTargetBitrateBps),
1838 DataRate::BitsPerSec(kTargetBitrateBps),
1839 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001840 video_stream_encoder_->SetSource(&video_source_,
1841 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001842 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1846
Jonathan Yubc771b72017-12-08 17:04:29 -08001847 // Adapt down as far as possible.
1848 rtc::VideoSinkWants last_wants;
1849 int64_t t = 1;
1850 int loop_count = 0;
1851 do {
1852 ++loop_count;
1853 last_wants = video_source_.sink_wants();
1854
1855 // Simulate the framerate we've been asked to adapt to.
1856 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1857 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1858 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1859 mock_stats.input_frame_rate = fps;
1860 stats_proxy_->SetMockStats(mock_stats);
1861
1862 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1863 sink_.WaitForEncodedFrame(t);
1864 t += frame_interval_ms;
1865
mflodmancc3d4422017-08-03 08:27:51 -07001866 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001867 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001868 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001869 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1870 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001871 } while (video_source_.sink_wants().max_pixel_count <
1872 last_wants.max_pixel_count ||
1873 video_source_.sink_wants().max_framerate_fps <
1874 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001875
Jonathan Yubc771b72017-12-08 17:04:29 -08001876 // Verify that we've adapted all the way down.
1877 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001878 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001879 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1880 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001881 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001882 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1883 *video_source_.last_sent_height());
1884 EXPECT_EQ(kMinBalancedFramerateFps,
1885 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001886
Jonathan Yubc771b72017-12-08 17:04:29 -08001887 // Adapt back up the same number of times we adapted down.
1888 for (int i = 0; i < loop_count - 1; ++i) {
1889 last_wants = video_source_.sink_wants();
1890
1891 // Simulate the framerate we've been asked to adapt to.
1892 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1893 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1894 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1895 mock_stats.input_frame_rate = fps;
1896 stats_proxy_->SetMockStats(mock_stats);
1897
1898 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1899 sink_.WaitForEncodedFrame(t);
1900 t += frame_interval_ms;
1901
Henrik Boström91aa7322020-04-28 12:24:33 +02001902 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001903 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001904 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001905 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1906 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001907 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1908 last_wants.max_pixel_count ||
1909 video_source_.sink_wants().max_framerate_fps >
1910 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001911 }
1912
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001913 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001914 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001915 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001916 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1917 EXPECT_EQ((loop_count - 1) * 2,
1918 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001919
mflodmancc3d4422017-08-03 08:27:51 -07001920 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001921}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001922
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001923TEST_F(VideoStreamEncoderTest,
1924 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1925 video_stream_encoder_->OnBitrateUpdated(
1926 DataRate::BitsPerSec(kTargetBitrateBps),
1927 DataRate::BitsPerSec(kTargetBitrateBps),
1928 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001929 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001930
1931 const int kFrameWidth = 1280;
1932 const int kFrameHeight = 720;
1933
1934 int64_t ntp_time = kFrameIntervalMs;
1935
1936 // Force an input frame rate to be available, or the adaptation call won't
1937 // know what framerate to adapt form.
1938 const int kInputFps = 30;
1939 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1940 stats.input_frame_rate = kInputFps;
1941 stats_proxy_->SetMockStats(stats);
1942
1943 video_source_.set_adaptation_enabled(true);
1944 video_stream_encoder_->SetSource(
1945 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001946 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001947 video_source_.IncomingCapturedFrame(
1948 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1949 sink_.WaitForEncodedFrame(ntp_time);
1950 ntp_time += kFrameIntervalMs;
1951
1952 // Trigger CPU overuse.
1953 video_stream_encoder_->TriggerCpuOveruse();
1954 video_source_.IncomingCapturedFrame(
1955 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1956 sink_.WaitForEncodedFrame(ntp_time);
1957 ntp_time += kFrameIntervalMs;
1958
1959 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1960 EXPECT_EQ(std::numeric_limits<int>::max(),
1961 video_source_.sink_wants().max_pixel_count);
1962 // Some framerate constraint should be set.
1963 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1964 EXPECT_LT(restricted_fps, kInputFps);
1965 video_source_.IncomingCapturedFrame(
1966 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1967 sink_.WaitForEncodedFrame(ntp_time);
1968 ntp_time += 100;
1969
Henrik Boström2671dac2020-05-19 16:29:09 +02001970 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001971 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
1972 // Give the encoder queue time to process the change in degradation preference
1973 // by waiting for an encoded frame.
1974 video_source_.IncomingCapturedFrame(
1975 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1976 sink_.WaitForEncodedFrame(ntp_time);
1977 ntp_time += kFrameIntervalMs;
1978
1979 video_stream_encoder_->TriggerQualityLow();
1980 video_source_.IncomingCapturedFrame(
1981 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1982 sink_.WaitForEncodedFrame(ntp_time);
1983 ntp_time += kFrameIntervalMs;
1984
1985 // Some resolution constraint should be set.
1986 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1987 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1988 kFrameWidth * kFrameHeight);
1989 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
1990
1991 int pixel_count = video_source_.sink_wants().max_pixel_count;
1992 // Triggering a CPU underuse should not change the sink wants since it has
1993 // not been overused for resolution since we changed degradation preference.
1994 video_stream_encoder_->TriggerCpuUnderuse();
1995 video_source_.IncomingCapturedFrame(
1996 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1997 sink_.WaitForEncodedFrame(ntp_time);
1998 ntp_time += kFrameIntervalMs;
1999 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2000 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2001
Evan Shrubsole64469032020-06-11 10:45:29 +02002002 // Change the degradation preference back. CPU underuse should not adapt since
2003 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002004 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002005 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2006 video_source_.IncomingCapturedFrame(
2007 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2008 sink_.WaitForEncodedFrame(ntp_time);
2009 ntp_time += 100;
2010 // Resolution adaptations is gone after changing degradation preference.
2011 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2012 EXPECT_EQ(std::numeric_limits<int>::max(),
2013 video_source_.sink_wants().max_pixel_count);
2014 // The fps adaptation from above is now back.
2015 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2016
2017 // Trigger CPU underuse.
2018 video_stream_encoder_->TriggerCpuUnderuse();
2019 video_source_.IncomingCapturedFrame(
2020 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2021 sink_.WaitForEncodedFrame(ntp_time);
2022 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002023 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2024
2025 // Trigger QP underuse, fps should return to normal.
2026 video_stream_encoder_->TriggerQualityHigh();
2027 video_source_.IncomingCapturedFrame(
2028 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2029 sink_.WaitForEncodedFrame(ntp_time);
2030 ntp_time += kFrameIntervalMs;
2031 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002032
2033 video_stream_encoder_->Stop();
2034}
2035
mflodmancc3d4422017-08-03 08:27:51 -07002036TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002038 DataRate::BitsPerSec(kTargetBitrateBps),
2039 DataRate::BitsPerSec(kTargetBitrateBps),
2040 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002041 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002042
sprangc5d62e22017-04-02 23:53:04 -07002043 const int kFrameWidth = 1280;
2044 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002045
Åsa Persson8c1bf952018-09-13 10:42:19 +02002046 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002047
kthelgason5e13d412016-12-01 03:59:51 -08002048 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002049 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002050 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002051 frame_timestamp += kFrameIntervalMs;
2052
perkj803d97f2016-11-01 11:45:46 -07002053 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002054 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002055 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002056 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002057 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002058 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002059
asapersson0944a802017-04-07 00:57:58 -07002060 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002061 // wanted resolution.
2062 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2063 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2064 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002065 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002066
2067 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002068 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002069 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002070 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002071 // Give the encoder queue time to process the change in degradation preference
2072 // by waiting for an encoded frame.
2073 new_video_source.IncomingCapturedFrame(
2074 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2075 sink_.WaitForEncodedFrame(frame_timestamp);
2076 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002077 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002078 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002079
sprangc5d62e22017-04-02 23:53:04 -07002080 // Force an input frame rate to be available, or the adaptation call won't
2081 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002082 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002083 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002084 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002085 stats_proxy_->SetMockStats(stats);
2086
mflodmancc3d4422017-08-03 08:27:51 -07002087 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002088 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002089 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002090 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002091 frame_timestamp += kFrameIntervalMs;
2092
2093 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002094 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002095 EXPECT_EQ(std::numeric_limits<int>::max(),
2096 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002097 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002098
asapersson02465b82017-04-10 01:12:52 -07002099 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002100 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2101 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002102 // Give the encoder queue time to process the change in degradation preference
2103 // by waiting for an encoded frame.
2104 new_video_source.IncomingCapturedFrame(
2105 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2106 sink_.WaitForEncodedFrame(frame_timestamp);
2107 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002108 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002109
mflodmancc3d4422017-08-03 08:27:51 -07002110 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002111 new_video_source.IncomingCapturedFrame(
2112 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002113 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002114 frame_timestamp += kFrameIntervalMs;
2115
2116 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002117 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002118
2119 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002120 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002121 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002122 // Give the encoder queue time to process the change in degradation preference
2123 // by waiting for an encoded frame.
2124 new_video_source.IncomingCapturedFrame(
2125 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2126 sink_.WaitForEncodedFrame(frame_timestamp);
2127 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002128 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2129 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002130 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002131 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002132
2133 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002134 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002135 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002136 // Give the encoder queue time to process the change in degradation preference
2137 // by waiting for an encoded frame.
2138 new_video_source.IncomingCapturedFrame(
2139 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2140 sink_.WaitForEncodedFrame(frame_timestamp);
2141 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002142 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2143 EXPECT_EQ(std::numeric_limits<int>::max(),
2144 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002145 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002146
mflodmancc3d4422017-08-03 08:27:51 -07002147 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002148}
2149
mflodmancc3d4422017-08-03 08:27:51 -07002150TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002151 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002152 DataRate::BitsPerSec(kTargetBitrateBps),
2153 DataRate::BitsPerSec(kTargetBitrateBps),
2154 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002155
asaperssonfab67072017-04-04 05:51:49 -07002156 const int kWidth = 1280;
2157 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002158 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002159 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002160 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2161 EXPECT_FALSE(stats.bw_limited_resolution);
2162 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2163
2164 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002165 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002166 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002167 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002168
2169 stats = stats_proxy_->GetStats();
2170 EXPECT_TRUE(stats.bw_limited_resolution);
2171 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2172
2173 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002174 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002175 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002176 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002177
2178 stats = stats_proxy_->GetStats();
2179 EXPECT_FALSE(stats.bw_limited_resolution);
2180 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2181 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2182
mflodmancc3d4422017-08-03 08:27:51 -07002183 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002184}
2185
mflodmancc3d4422017-08-03 08:27:51 -07002186TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002187 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002188 DataRate::BitsPerSec(kTargetBitrateBps),
2189 DataRate::BitsPerSec(kTargetBitrateBps),
2190 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002191
2192 const int kWidth = 1280;
2193 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002194 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002195 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002196 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2197 EXPECT_FALSE(stats.cpu_limited_resolution);
2198 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2199
2200 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002201 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002202 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002203 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002204
2205 stats = stats_proxy_->GetStats();
2206 EXPECT_TRUE(stats.cpu_limited_resolution);
2207 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2208
2209 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002210 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002211 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002212 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002213
2214 stats = stats_proxy_->GetStats();
2215 EXPECT_FALSE(stats.cpu_limited_resolution);
2216 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002217 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002218
mflodmancc3d4422017-08-03 08:27:51 -07002219 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002220}
2221
mflodmancc3d4422017-08-03 08:27:51 -07002222TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002224 DataRate::BitsPerSec(kTargetBitrateBps),
2225 DataRate::BitsPerSec(kTargetBitrateBps),
2226 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002227
asaperssonfab67072017-04-04 05:51:49 -07002228 const int kWidth = 1280;
2229 const int kHeight = 720;
2230 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002231 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002232 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002233 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002234 EXPECT_FALSE(stats.cpu_limited_resolution);
2235 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2236
asaperssonfab67072017-04-04 05:51:49 -07002237 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002238 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002239 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002240 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002241 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002242 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002243 EXPECT_TRUE(stats.cpu_limited_resolution);
2244 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2245
2246 // Set new source with adaptation still enabled.
2247 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002248 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002249 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002250
asaperssonfab67072017-04-04 05:51:49 -07002251 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002252 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002253 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002254 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002255 EXPECT_TRUE(stats.cpu_limited_resolution);
2256 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2257
2258 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002259 video_stream_encoder_->SetSource(&new_video_source,
2260 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002261
asaperssonfab67072017-04-04 05:51:49 -07002262 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002263 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002264 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002265 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002266 EXPECT_FALSE(stats.cpu_limited_resolution);
2267 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2268
2269 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002270 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002271 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002272
asaperssonfab67072017-04-04 05:51:49 -07002273 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002274 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002275 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002276 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002277 EXPECT_TRUE(stats.cpu_limited_resolution);
2278 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2279
asaperssonfab67072017-04-04 05:51:49 -07002280 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002281 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002282 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002283 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002284 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002285 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002286 EXPECT_FALSE(stats.cpu_limited_resolution);
2287 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002288 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002289
mflodmancc3d4422017-08-03 08:27:51 -07002290 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002291}
2292
mflodmancc3d4422017-08-03 08:27:51 -07002293TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002294 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002295 DataRate::BitsPerSec(kTargetBitrateBps),
2296 DataRate::BitsPerSec(kTargetBitrateBps),
2297 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002298
asaperssonfab67072017-04-04 05:51:49 -07002299 const int kWidth = 1280;
2300 const int kHeight = 720;
2301 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002302 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002303 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002304 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002305 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002306 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002307
2308 // Set new source with adaptation still enabled.
2309 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002310 video_stream_encoder_->SetSource(&new_video_source,
2311 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002312
asaperssonfab67072017-04-04 05:51:49 -07002313 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002314 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002315 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002316 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002317 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002318 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002319
asaperssonfab67072017-04-04 05:51:49 -07002320 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002321 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002322 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002323 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002324 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002325 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002326 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002327 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002328
asaperssonfab67072017-04-04 05:51:49 -07002329 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002330 video_stream_encoder_->SetSource(&new_video_source,
2331 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002332
asaperssonfab67072017-04-04 05:51:49 -07002333 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002334 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002335 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002336 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002337 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002338 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002339
asapersson02465b82017-04-10 01:12:52 -07002340 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002341 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002342 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002343
asaperssonfab67072017-04-04 05:51:49 -07002344 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002345 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002346 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002347 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002348 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002349 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2350 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002351
mflodmancc3d4422017-08-03 08:27:51 -07002352 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002353}
2354
mflodmancc3d4422017-08-03 08:27:51 -07002355TEST_F(VideoStreamEncoderTest,
2356 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002357 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002358 DataRate::BitsPerSec(kTargetBitrateBps),
2359 DataRate::BitsPerSec(kTargetBitrateBps),
2360 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002361
2362 const int kWidth = 1280;
2363 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002364 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002365 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002366 video_source_.IncomingCapturedFrame(
2367 CreateFrame(timestamp_ms, kWidth, kHeight));
2368 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002369 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2370 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2371 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2372
2373 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002374 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002375 timestamp_ms += kFrameIntervalMs;
2376 video_source_.IncomingCapturedFrame(
2377 CreateFrame(timestamp_ms, kWidth, kHeight));
2378 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002379 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2380 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2381 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2382
2383 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002384 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002385 timestamp_ms += kFrameIntervalMs;
2386 video_source_.IncomingCapturedFrame(
2387 CreateFrame(timestamp_ms, kWidth, kHeight));
2388 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002389 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2390 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2391 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2392
Niels Möller4db138e2018-04-19 09:04:13 +02002393 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002394 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002395
2396 VideoEncoderConfig video_encoder_config;
2397 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2398 // Make format different, to force recreation of encoder.
2399 video_encoder_config.video_format.parameters["foo"] = "foo";
2400 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002401 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002402 timestamp_ms += kFrameIntervalMs;
2403 video_source_.IncomingCapturedFrame(
2404 CreateFrame(timestamp_ms, kWidth, kHeight));
2405 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002406 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2407 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2408 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2409
mflodmancc3d4422017-08-03 08:27:51 -07002410 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002411}
2412
mflodmancc3d4422017-08-03 08:27:51 -07002413TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002414 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002416 DataRate::BitsPerSec(kTargetBitrateBps),
2417 DataRate::BitsPerSec(kTargetBitrateBps),
2418 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2419
2420 const int kWidth = 1280;
2421 const int kHeight = 720;
2422 int sequence = 1;
2423
2424 // Enable BALANCED preference, no initial limitation.
2425 test::FrameForwarder source;
2426 video_stream_encoder_->SetSource(&source,
2427 webrtc::DegradationPreference::BALANCED);
2428 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2429 WaitForEncodedFrame(sequence++);
2430 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2431 EXPECT_FALSE(stats.cpu_limited_resolution);
2432 EXPECT_FALSE(stats.cpu_limited_framerate);
2433 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2434
2435 // Trigger CPU overuse, should now adapt down.
2436 video_stream_encoder_->TriggerCpuOveruse();
2437 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2438 WaitForEncodedFrame(sequence++);
2439 stats = stats_proxy_->GetStats();
2440 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2441
2442 // Set new degradation preference should clear restrictions since we changed
2443 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002444 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002445 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2446 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2447 WaitForEncodedFrame(sequence++);
2448 stats = stats_proxy_->GetStats();
2449 EXPECT_FALSE(stats.cpu_limited_resolution);
2450 EXPECT_FALSE(stats.cpu_limited_framerate);
2451 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2452
2453 // Force an input frame rate to be available, or the adaptation call won't
2454 // know what framerate to adapt from.
2455 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2456 mock_stats.input_frame_rate = 30;
2457 stats_proxy_->SetMockStats(mock_stats);
2458 video_stream_encoder_->TriggerCpuOveruse();
2459 stats_proxy_->ResetMockStats();
2460 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2461 WaitForEncodedFrame(sequence++);
2462
2463 // We have now adapted once.
2464 stats = stats_proxy_->GetStats();
2465 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2466
2467 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002468 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2469 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002470 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2471 WaitForEncodedFrame(sequence++);
2472 stats = stats_proxy_->GetStats();
2473 EXPECT_FALSE(stats.cpu_limited_resolution);
2474 EXPECT_FALSE(stats.cpu_limited_framerate);
2475 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2476
2477 video_stream_encoder_->Stop();
2478}
2479
2480TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002481 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002483 DataRate::BitsPerSec(kTargetBitrateBps),
2484 DataRate::BitsPerSec(kTargetBitrateBps),
2485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002486
asapersson0944a802017-04-07 00:57:58 -07002487 const int kWidth = 1280;
2488 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002489 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002490
asaperssonfab67072017-04-04 05:51:49 -07002491 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002492 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002493 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002494 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002495 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002496 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2497
asapersson02465b82017-04-10 01:12:52 -07002498 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002499 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002500 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002501 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002502 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002503 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002504 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002505 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2506
2507 // Set new source with adaptation still enabled.
2508 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002509 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002510 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002511
2512 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002513 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002514 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002515 stats = stats_proxy_->GetStats();
2516 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002517 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002518 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2519
sprangc5d62e22017-04-02 23:53:04 -07002520 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002521 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002522 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002523 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002524 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002525 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002526 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002527 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002528 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002529 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002530 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2531
sprangc5d62e22017-04-02 23:53:04 -07002532 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002533 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002534 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2535 mock_stats.input_frame_rate = 30;
2536 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002537 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002538 stats_proxy_->ResetMockStats();
2539
2540 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002541 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002542 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002543
2544 // Framerate now adapted.
2545 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002546 EXPECT_FALSE(stats.cpu_limited_resolution);
2547 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002548 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2549
2550 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002551 video_stream_encoder_->SetSource(&new_video_source,
2552 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002553 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002554 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002555 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002556
2557 stats = stats_proxy_->GetStats();
2558 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002559 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002560 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2561
2562 // Try to trigger overuse. Should not succeed.
2563 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002564 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002565 stats_proxy_->ResetMockStats();
2566
2567 stats = stats_proxy_->GetStats();
2568 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002569 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002570 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2571
2572 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002573 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002574 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002575 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002576 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002577 stats = stats_proxy_->GetStats();
2578 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002579 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002580 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002581
2582 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002583 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002584 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002585 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002586 stats = stats_proxy_->GetStats();
2587 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002588 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002589 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2590
2591 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002592 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002593 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002594 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002595 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002596 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002597 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002598 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002599 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002600 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002601 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2602
2603 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002604 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002605 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002606 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002607 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002608 stats = stats_proxy_->GetStats();
2609 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002610 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002611 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002612 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002613
mflodmancc3d4422017-08-03 08:27:51 -07002614 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002615}
2616
mflodmancc3d4422017-08-03 08:27:51 -07002617TEST_F(VideoStreamEncoderTest,
2618 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002619 const int kWidth = 1280;
2620 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002622 DataRate::BitsPerSec(kTargetBitrateBps),
2623 DataRate::BitsPerSec(kTargetBitrateBps),
2624 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002625
asaperssonfab67072017-04-04 05:51:49 -07002626 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002627 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002628
asaperssonfab67072017-04-04 05:51:49 -07002629 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002630 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002631
asaperssonfab67072017-04-04 05:51:49 -07002632 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002633 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002634
asaperssonfab67072017-04-04 05:51:49 -07002635 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002636 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002637
kthelgason876222f2016-11-29 01:44:11 -08002638 // Expect a scale down.
2639 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002640 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002641
asapersson02465b82017-04-10 01:12:52 -07002642 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002643 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002644 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002645 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002646
asaperssonfab67072017-04-04 05:51:49 -07002647 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002648 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002649 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002650 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002651
asaperssonfab67072017-04-04 05:51:49 -07002652 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002653 EXPECT_EQ(std::numeric_limits<int>::max(),
2654 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002655
asaperssonfab67072017-04-04 05:51:49 -07002656 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002657 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002658 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002659 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002660
asapersson02465b82017-04-10 01:12:52 -07002661 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002662 EXPECT_EQ(std::numeric_limits<int>::max(),
2663 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002664
mflodmancc3d4422017-08-03 08:27:51 -07002665 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002666}
2667
mflodmancc3d4422017-08-03 08:27:51 -07002668TEST_F(VideoStreamEncoderTest,
2669 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002670 const int kWidth = 1280;
2671 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002673 DataRate::BitsPerSec(kTargetBitrateBps),
2674 DataRate::BitsPerSec(kTargetBitrateBps),
2675 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002676
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002677 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002678 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002679 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002680 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002681
2682 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002683 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002684 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002685 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2686 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2687
2688 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002689 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002690 EXPECT_THAT(source.sink_wants(),
2691 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002692 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2693 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2694 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2695
2696 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002697 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002698 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2699 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2700 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2701
mflodmancc3d4422017-08-03 08:27:51 -07002702 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002703}
2704
mflodmancc3d4422017-08-03 08:27:51 -07002705TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002706 const int kWidth = 1280;
2707 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002709 DataRate::BitsPerSec(kTargetBitrateBps),
2710 DataRate::BitsPerSec(kTargetBitrateBps),
2711 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002712
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002713 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002714 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002715 video_stream_encoder_->SetSource(&source,
2716 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002717 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2718 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002719 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002720
2721 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002722 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002723 EXPECT_THAT(source.sink_wants(),
2724 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002725 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2726 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2727 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2728
2729 // Trigger adapt down for same input resolution, expect no change.
2730 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2731 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002732 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002733 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2734 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2735 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2736
2737 // Trigger adapt down for larger input resolution, expect no change.
2738 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2739 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002740 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002741 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2742 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2743 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2744
mflodmancc3d4422017-08-03 08:27:51 -07002745 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002746}
2747
mflodmancc3d4422017-08-03 08:27:51 -07002748TEST_F(VideoStreamEncoderTest,
2749 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002750 const int kWidth = 1280;
2751 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002753 DataRate::BitsPerSec(kTargetBitrateBps),
2754 DataRate::BitsPerSec(kTargetBitrateBps),
2755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002756
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002757 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002758 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002759 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002760 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002761
2762 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002763 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002764 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002765 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2766 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2767
2768 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002769 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002770 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002771 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2772 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2773
mflodmancc3d4422017-08-03 08:27:51 -07002774 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002775}
2776
mflodmancc3d4422017-08-03 08:27:51 -07002777TEST_F(VideoStreamEncoderTest,
2778 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002779 const int kWidth = 1280;
2780 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002782 DataRate::BitsPerSec(kTargetBitrateBps),
2783 DataRate::BitsPerSec(kTargetBitrateBps),
2784 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002785
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002786 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002787 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002788 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002789 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002790
2791 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002792 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002793 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002794 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002795 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2796
2797 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002798 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002799 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002800 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002801 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2802
mflodmancc3d4422017-08-03 08:27:51 -07002803 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002804}
2805
mflodmancc3d4422017-08-03 08:27:51 -07002806TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002807 const int kWidth = 1280;
2808 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002810 DataRate::BitsPerSec(kTargetBitrateBps),
2811 DataRate::BitsPerSec(kTargetBitrateBps),
2812 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002813
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002814 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002815 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002816 video_stream_encoder_->SetSource(&source,
2817 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002818
2819 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2820 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002821 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002822 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2823 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2824 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2825
2826 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002827 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002828 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2830 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2831 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2832
mflodmancc3d4422017-08-03 08:27:51 -07002833 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002834}
2835
mflodmancc3d4422017-08-03 08:27:51 -07002836TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002837 const int kWidth = 1280;
2838 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002840 DataRate::BitsPerSec(kTargetBitrateBps),
2841 DataRate::BitsPerSec(kTargetBitrateBps),
2842 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002843
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002844 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002845 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002846 video_stream_encoder_->SetSource(&source,
2847 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002848
2849 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2850 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002851 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2854 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2855
2856 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002858 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002859 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2860 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2861 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2862
mflodmancc3d4422017-08-03 08:27:51 -07002863 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002864}
2865
mflodmancc3d4422017-08-03 08:27:51 -07002866TEST_F(VideoStreamEncoderTest,
2867 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002868 const int kWidth = 1280;
2869 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002871 DataRate::BitsPerSec(kTargetBitrateBps),
2872 DataRate::BitsPerSec(kTargetBitrateBps),
2873 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002874
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002875 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002876 AdaptingFrameForwarder source;
2877 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002878 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002879 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002880
2881 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002883 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002884 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2885 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2886
2887 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002888 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002889 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002890 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002891 EXPECT_THAT(source.sink_wants(),
2892 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002893 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2894 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2895
2896 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002897 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002898 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002899 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2900 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2901 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2902
mflodmancc3d4422017-08-03 08:27:51 -07002903 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002904}
2905
mflodmancc3d4422017-08-03 08:27:51 -07002906TEST_F(VideoStreamEncoderTest,
2907 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002908 const int kWidth = 1280;
2909 const int kHeight = 720;
2910 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002911 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002912 DataRate::BitsPerSec(kTargetBitrateBps),
2913 DataRate::BitsPerSec(kTargetBitrateBps),
2914 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002915
2916 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2917 stats.input_frame_rate = kInputFps;
2918 stats_proxy_->SetMockStats(stats);
2919
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002920 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002921 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2922 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002923 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002924
2925 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002926 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002927 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2928 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002929 EXPECT_THAT(video_source_.sink_wants(),
2930 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002931
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002932 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002933 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002934 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002935 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002936 // Give the encoder queue time to process the change in degradation preference
2937 // by waiting for an encoded frame.
2938 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2939 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002940 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002941
2942 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002943 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002944 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2945 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002946 EXPECT_THAT(new_video_source.sink_wants(),
2947 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002948
2949 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002951 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002952
mflodmancc3d4422017-08-03 08:27:51 -07002953 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002954}
2955
mflodmancc3d4422017-08-03 08:27:51 -07002956TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002957 const int kWidth = 1280;
2958 const int kHeight = 720;
2959 const size_t kNumFrames = 10;
2960
Henrik Boström381d1092020-05-12 18:49:07 +02002961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002962 DataRate::BitsPerSec(kTargetBitrateBps),
2963 DataRate::BitsPerSec(kTargetBitrateBps),
2964 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002965
asaperssond0de2952017-04-21 01:47:31 -07002966 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002967 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002968 video_source_.set_adaptation_enabled(true);
2969
2970 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2971 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2972
2973 int downscales = 0;
2974 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002975 video_source_.IncomingCapturedFrame(
2976 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2977 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002978
asaperssonfab67072017-04-04 05:51:49 -07002979 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002980 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002981 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002982 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002983
2984 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2985 ++downscales;
2986
2987 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2988 EXPECT_EQ(downscales,
2989 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2990 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002991 }
mflodmancc3d4422017-08-03 08:27:51 -07002992 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002993}
2994
mflodmancc3d4422017-08-03 08:27:51 -07002995TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002996 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2997 const int kWidth = 1280;
2998 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003000 DataRate::BitsPerSec(kTargetBitrateBps),
3001 DataRate::BitsPerSec(kTargetBitrateBps),
3002 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003003
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003004 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003005 AdaptingFrameForwarder source;
3006 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003007 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003008 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003009
Åsa Persson8c1bf952018-09-13 10:42:19 +02003010 int64_t timestamp_ms = kFrameIntervalMs;
3011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003012 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003013 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003014 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3015 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3016
3017 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003018 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003019 timestamp_ms += kFrameIntervalMs;
3020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3021 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003022 EXPECT_THAT(source.sink_wants(),
3023 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003024 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3025 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3026
3027 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003028 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003029 timestamp_ms += kFrameIntervalMs;
3030 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003031 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003032 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003033 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3034 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3035
3036 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003037 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003038 timestamp_ms += kFrameIntervalMs;
3039 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3040 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003041 EXPECT_THAT(source.sink_wants(),
3042 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003043 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3044 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3045
3046 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003047 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003048 timestamp_ms += kFrameIntervalMs;
3049 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003050 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003051 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003052 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3053 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3054
mflodmancc3d4422017-08-03 08:27:51 -07003055 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003056}
3057
mflodmancc3d4422017-08-03 08:27:51 -07003058TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003059 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3060 const int kWidth = 1280;
3061 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003063 DataRate::BitsPerSec(kTargetBitrateBps),
3064 DataRate::BitsPerSec(kTargetBitrateBps),
3065 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003066
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003067 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003068 AdaptingFrameForwarder source;
3069 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003070 video_stream_encoder_->SetSource(&source,
3071 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003072
Åsa Persson8c1bf952018-09-13 10:42:19 +02003073 int64_t timestamp_ms = kFrameIntervalMs;
3074 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003075 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003076 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003077 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3078 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3079
3080 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003081 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003082 timestamp_ms += kFrameIntervalMs;
3083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3084 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003085 EXPECT_THAT(source.sink_wants(),
3086 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003087 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3088 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3089
3090 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003091 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003092 timestamp_ms += kFrameIntervalMs;
3093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003094 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003095 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003096 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3097 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3098
3099 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003100 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003101 timestamp_ms += kFrameIntervalMs;
3102 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3103 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003104 EXPECT_THAT(source.sink_wants(),
3105 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003106 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3107 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3108
3109 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003110 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003111 timestamp_ms += kFrameIntervalMs;
3112 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003113 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003114 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003115 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3116 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3117
mflodmancc3d4422017-08-03 08:27:51 -07003118 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003119}
3120
Sergey Silkin41c650b2019-10-14 13:12:19 +02003121TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3122 fake_encoder_.SetResolutionBitrateLimits(
3123 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3124
Henrik Boström381d1092020-05-12 18:49:07 +02003125 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003126 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3127 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3128 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3129 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003130
3131 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3132 AdaptingFrameForwarder source;
3133 source.set_adaptation_enabled(true);
3134 video_stream_encoder_->SetSource(
3135 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3136
3137 // Insert 720p frame.
3138 int64_t timestamp_ms = kFrameIntervalMs;
3139 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3140 WaitForEncodedFrame(1280, 720);
3141
3142 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003143 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003144 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3145 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3146 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3147 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003148 video_stream_encoder_->TriggerQualityLow();
3149
3150 // Insert 720p frame. It should be downscaled and encoded.
3151 timestamp_ms += kFrameIntervalMs;
3152 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3153 WaitForEncodedFrame(960, 540);
3154
3155 // Trigger adapt up. Higher resolution should not be requested duo to lack
3156 // of bitrate.
3157 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003158 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003159
3160 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003161 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003162 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3163 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3164 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3165 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003166
3167 // Trigger adapt up. Higher resolution should be requested.
3168 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003169 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003170
3171 video_stream_encoder_->Stop();
3172}
3173
3174TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3175 fake_encoder_.SetResolutionBitrateLimits(
3176 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3177
3178 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003180 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3181 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3182 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3183 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003184
3185 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3186 AdaptingFrameForwarder source;
3187 source.set_adaptation_enabled(true);
3188 video_stream_encoder_->SetSource(
3189 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3190
3191 // Insert 720p frame. It should be dropped and lower resolution should be
3192 // requested.
3193 int64_t timestamp_ms = kFrameIntervalMs;
3194 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3195 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003196 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003197
3198 // Insert 720p frame. It should be downscaled and encoded.
3199 timestamp_ms += kFrameIntervalMs;
3200 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3201 WaitForEncodedFrame(960, 540);
3202
3203 video_stream_encoder_->Stop();
3204}
3205
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003206class BalancedDegradationTest : public VideoStreamEncoderTest {
3207 protected:
3208 void SetupTest() {
3209 // Reset encoder for field trials to take effect.
3210 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003211 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003212
3213 // Enable BALANCED preference.
3214 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003215 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3216 }
3217
3218 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003220 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3221 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003222 }
3223
Åsa Persson45b176f2019-09-30 11:19:05 +02003224 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003225 timestamp_ms_ += kFrameIntervalMs;
3226 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003227 }
3228
3229 void InsertFrameAndWaitForEncoded() {
3230 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003231 sink_.WaitForEncodedFrame(timestamp_ms_);
3232 }
3233
3234 const int kWidth = 640; // pixels:640x360=230400
3235 const int kHeight = 360;
3236 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3237 int64_t timestamp_ms_ = 0;
3238 AdaptingFrameForwarder source_;
3239};
3240
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003241TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003242 test::ScopedFieldTrials field_trials(
3243 "WebRTC-Video-BalancedDegradationSettings/"
3244 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3245 SetupTest();
3246
3247 // Force input frame rate.
3248 const int kInputFps = 24;
3249 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3250 stats.input_frame_rate = kInputFps;
3251 stats_proxy_->SetMockStats(stats);
3252
Åsa Persson45b176f2019-09-30 11:19:05 +02003253 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003254 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003255
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003256 // Trigger adapt down, expect scaled down framerate and resolution,
3257 // since Fps diff (input-requested:0) < threshold.
3258 video_stream_encoder_->TriggerQualityLow();
3259 EXPECT_THAT(source_.sink_wants(),
3260 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003261
3262 video_stream_encoder_->Stop();
3263}
3264
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003265TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003266 test::ScopedFieldTrials field_trials(
3267 "WebRTC-Video-BalancedDegradationSettings/"
3268 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3269 SetupTest();
3270
3271 // Force input frame rate.
3272 const int kInputFps = 25;
3273 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3274 stats.input_frame_rate = kInputFps;
3275 stats_proxy_->SetMockStats(stats);
3276
Åsa Persson45b176f2019-09-30 11:19:05 +02003277 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003278 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003279
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003280 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3281 // Fps diff (input-requested:1) == threshold.
3282 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003283 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003284
3285 video_stream_encoder_->Stop();
3286}
3287
3288TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3289 test::ScopedFieldTrials field_trials(
3290 "WebRTC-Video-BalancedDegradationSettings/"
3291 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3292 SetupTest();
3293
3294 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3295
Åsa Persson45b176f2019-09-30 11:19:05 +02003296 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003297 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003298
3299 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3300 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003301 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003302
3303 video_stream_encoder_->Stop();
3304}
3305
Åsa Perssonccfb3402019-09-25 15:13:04 +02003306TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003307 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003308 "WebRTC-Video-BalancedDegradationSettings/"
3309 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003310 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003311
Åsa Persson1b247f12019-08-14 17:26:39 +02003312 const int kMinBitrateBps = 425000;
3313 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003314 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003315
Åsa Persson45b176f2019-09-30 11:19:05 +02003316 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003317 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003318 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3319
3320 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3321 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003323 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003324 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3325
3326 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3327 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003328 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003329 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003330 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3331
Åsa Persson30ab0152019-08-27 12:22:33 +02003332 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3333 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003334 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003335 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003336 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003337 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3338
3339 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003340 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003341 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003342 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003343
Åsa Persson30ab0152019-08-27 12:22:33 +02003344 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003345 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003346 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003347 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003348 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003349 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3350
3351 video_stream_encoder_->Stop();
3352}
3353
Åsa Perssonccfb3402019-09-25 15:13:04 +02003354TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003355 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3356 test::ScopedFieldTrials field_trials(
3357 "WebRTC-Video-BalancedDegradationSettings/"
3358 "pixels:57600|129600|230400,fps:7|24|24/");
3359 SetupTest();
3360 OnBitrateUpdated(kLowTargetBitrateBps);
3361
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003362 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003363
3364 // Insert frame, expect scaled down:
3365 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3366 InsertFrame();
3367 EXPECT_FALSE(WaitForFrame(1000));
3368 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3369 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3370
3371 // Insert frame, expect scaled down:
3372 // resolution (320x180@24fps).
3373 InsertFrame();
3374 EXPECT_FALSE(WaitForFrame(1000));
3375 EXPECT_LT(source_.sink_wants().max_pixel_count,
3376 source_.last_wants().max_pixel_count);
3377 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3378
3379 // Frame should not be dropped (min pixels per frame reached).
3380 InsertFrameAndWaitForEncoded();
3381
3382 video_stream_encoder_->Stop();
3383}
3384
3385TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003386 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003387 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003388 "WebRTC-Video-BalancedDegradationSettings/"
3389 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003390 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003391
Åsa Persson30ab0152019-08-27 12:22:33 +02003392 const int kResolutionMinBitrateBps = 435000;
3393 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003394 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003395
Åsa Persson45b176f2019-09-30 11:19:05 +02003396 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003397 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003398 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3401 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003402 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003403 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003404 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3405
3406 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3407 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003408 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003409 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003410 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3411
3412 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3413 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003414 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003415 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003416 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3417
Åsa Persson30ab0152019-08-27 12:22:33 +02003418 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3419 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003420 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003421 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003422 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3425 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003426 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003427 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3428
3429 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003430 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003431 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003432 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003433 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003434 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
3436 video_stream_encoder_->Stop();
3437}
3438
Åsa Perssonccfb3402019-09-25 15:13:04 +02003439TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003440 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003441 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003442 "WebRTC-Video-BalancedDegradationSettings/"
3443 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003444 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003445
Åsa Persson30ab0152019-08-27 12:22:33 +02003446 const int kMinBitrateBps = 425000;
3447 const int kTooLowMinBitrateBps = 424000;
3448 const int kResolutionMinBitrateBps = 435000;
3449 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003450 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003451
Åsa Persson45b176f2019-09-30 11:19:05 +02003452 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003453 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003454 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3455
3456 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3457 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003458 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003459 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003460 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3461
3462 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3463 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003464 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003465 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003466 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3467
3468 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3469 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003470 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003471 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003472 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3473
3474 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3475 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003476 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003477 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3478
3479 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003480 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003481 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003482 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003483 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003484 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3485
3486 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003487 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003488 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003489 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003490 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3491
3492 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003493 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003494 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003495 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003496 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003497 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3498
Åsa Persson1b247f12019-08-14 17:26:39 +02003499 video_stream_encoder_->Stop();
3500}
3501
mflodmancc3d4422017-08-03 08:27:51 -07003502TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003503 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3504 const int kWidth = 1280;
3505 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003507 DataRate::BitsPerSec(kTargetBitrateBps),
3508 DataRate::BitsPerSec(kTargetBitrateBps),
3509 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003510
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003511 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003512 AdaptingFrameForwarder source;
3513 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003514 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003515 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003516
Åsa Persson8c1bf952018-09-13 10:42:19 +02003517 int64_t timestamp_ms = kFrameIntervalMs;
3518 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003519 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003520 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003521 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3522 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3523 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3524 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3525
3526 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003527 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003528 timestamp_ms += kFrameIntervalMs;
3529 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3530 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003531 EXPECT_THAT(source.sink_wants(),
3532 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003533 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3535 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3536 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3537
3538 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003539 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003540 timestamp_ms += kFrameIntervalMs;
3541 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3542 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003543 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003544 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3546 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3547 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3548
Jonathan Yubc771b72017-12-08 17:04:29 -08003549 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003550 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003551 timestamp_ms += kFrameIntervalMs;
3552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3553 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003554 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003555 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3556 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003557 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003558 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3559
Jonathan Yubc771b72017-12-08 17:04:29 -08003560 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003561 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003562 timestamp_ms += kFrameIntervalMs;
3563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003565 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003566 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003567 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3569 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3570 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3571
Jonathan Yubc771b72017-12-08 17:04:29 -08003572 // Trigger quality adapt down, expect no change (min resolution reached).
3573 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003574 timestamp_ms += kFrameIntervalMs;
3575 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3576 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003577 EXPECT_THAT(source.sink_wants(), FpsMax());
3578 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003579 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3581 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3582 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3583
Evan Shrubsole64469032020-06-11 10:45:29 +02003584 // Trigger quality adapt up, expect upscaled resolution (480x270).
3585 video_stream_encoder_->TriggerQualityHigh();
3586 timestamp_ms += kFrameIntervalMs;
3587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3588 WaitForEncodedFrame(timestamp_ms);
3589 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3590 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3592 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3593 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3594
3595 // Trigger quality and cpu adapt up since both are most limited, expect
3596 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003597 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003598 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003599 timestamp_ms += kFrameIntervalMs;
3600 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3601 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003602 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003603 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3605 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003606 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003607
Evan Shrubsole64469032020-06-11 10:45:29 +02003608 // Trigger quality and cpu adapt up since both are most limited, expect
3609 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003610 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003611 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003612 timestamp_ms += kFrameIntervalMs;
3613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003615 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003616 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003617 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003619 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3620 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003621
Evan Shrubsole64469032020-06-11 10:45:29 +02003622 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3623 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003624 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003625 timestamp_ms += kFrameIntervalMs;
3626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3627 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003628 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003631 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003632 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003633
3634 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003635 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003636 timestamp_ms += kFrameIntervalMs;
3637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003638 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003639 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003640 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003641 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3642 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003643 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003644 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003645
mflodmancc3d4422017-08-03 08:27:51 -07003646 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003647}
3648
mflodmancc3d4422017-08-03 08:27:51 -07003649TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003650 const int kWidth = 640;
3651 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003652
Henrik Boström381d1092020-05-12 18:49:07 +02003653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003654 DataRate::BitsPerSec(kTargetBitrateBps),
3655 DataRate::BitsPerSec(kTargetBitrateBps),
3656 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003657
perkj803d97f2016-11-01 11:45:46 -07003658 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003659 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003660 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003661 }
3662
mflodmancc3d4422017-08-03 08:27:51 -07003663 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003664 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003665 video_source_.IncomingCapturedFrame(CreateFrame(
3666 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003667 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003668 }
3669
mflodmancc3d4422017-08-03 08:27:51 -07003670 video_stream_encoder_->Stop();
3671 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003672 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003673
Ying Wangef3998f2019-12-09 13:06:53 +01003674 EXPECT_METRIC_EQ(
3675 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3676 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003677 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3678}
3679
mflodmancc3d4422017-08-03 08:27:51 -07003680TEST_F(VideoStreamEncoderTest,
3681 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003683 DataRate::BitsPerSec(kTargetBitrateBps),
3684 DataRate::BitsPerSec(kTargetBitrateBps),
3685 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003686 const int kWidth = 640;
3687 const int kHeight = 360;
3688
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003689 video_stream_encoder_->SetSource(&video_source_,
3690 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003691
3692 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3693 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003694 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003695 }
3696
mflodmancc3d4422017-08-03 08:27:51 -07003697 video_stream_encoder_->Stop();
3698 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003699 stats_proxy_.reset();
3700
3701 EXPECT_EQ(0,
3702 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3703}
3704
mflodmancc3d4422017-08-03 08:27:51 -07003705TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003706 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003707 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003708
3709 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003710 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003711 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003712 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3713 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003714
sprang57c2fff2017-01-16 06:24:02 -08003715 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003716 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003718 DataRate::BitsPerSec(kLowTargetBitrateBps),
3719 DataRate::BitsPerSec(kLowTargetBitrateBps),
3720 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003721
sprang57c2fff2017-01-16 06:24:02 -08003722 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003723 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3724 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003725 VideoBitrateAllocation bitrate_allocation =
3726 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003727 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003728 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003729 // TODO(srte): The use of millisecs here looks like an error, but the tests
3730 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003731 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003732
3733 // Not called on second frame.
3734 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3735 .Times(0);
3736 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003737 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3738 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003739 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003740
3741 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003742 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3743 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003744 const int64_t start_time_ms = rtc::TimeMillis();
3745 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3746 video_source_.IncomingCapturedFrame(
3747 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3748 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003749 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003750 }
3751
3752 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003753 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003754
mflodmancc3d4422017-08-03 08:27:51 -07003755 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003756}
3757
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003758TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3759 // 2 TLs configured, temporal layers supported by encoder.
3760 const int kNumTemporalLayers = 2;
3761 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3762 fake_encoder_.SetTemporalLayersSupported(0, true);
3763
3764 // Bitrate allocated across temporal layers.
3765 const int kTl0Bps = kTargetBitrateBps *
3766 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003767 kNumTemporalLayers, /*temporal_id*/ 0,
3768 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003769 const int kTl1Bps = kTargetBitrateBps *
3770 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003771 kNumTemporalLayers, /*temporal_id*/ 1,
3772 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003773 VideoBitrateAllocation expected_bitrate;
3774 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3775 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3776
3777 VerifyAllocatedBitrate(expected_bitrate);
3778 video_stream_encoder_->Stop();
3779}
3780
3781TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3782 // 2 TLs configured, temporal layers not supported by encoder.
3783 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3784 fake_encoder_.SetTemporalLayersSupported(0, false);
3785
3786 // Temporal layers not supported by the encoder.
3787 // Total bitrate should be at ti:0.
3788 VideoBitrateAllocation expected_bitrate;
3789 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3790
3791 VerifyAllocatedBitrate(expected_bitrate);
3792 video_stream_encoder_->Stop();
3793}
3794
3795TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3796 // 2 TLs configured, temporal layers only supported for first stream.
3797 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3798 fake_encoder_.SetTemporalLayersSupported(0, true);
3799 fake_encoder_.SetTemporalLayersSupported(1, false);
3800
3801 const int kS0Bps = 150000;
3802 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003803 kS0Bps *
3804 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3805 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003806 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003807 kS0Bps *
3808 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3809 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003810 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3811 // Temporal layers not supported by si:1.
3812 VideoBitrateAllocation expected_bitrate;
3813 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3814 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3815 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3816
3817 VerifyAllocatedBitrate(expected_bitrate);
3818 video_stream_encoder_->Stop();
3819}
3820
Niels Möller7dc26b72017-12-06 10:27:48 +01003821TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3822 const int kFrameWidth = 1280;
3823 const int kFrameHeight = 720;
3824 const int kFramerate = 24;
3825
Henrik Boström381d1092020-05-12 18:49:07 +02003826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003827 DataRate::BitsPerSec(kTargetBitrateBps),
3828 DataRate::BitsPerSec(kTargetBitrateBps),
3829 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003830 test::FrameForwarder source;
3831 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003832 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003833
3834 // Insert a single frame, triggering initial configuration.
3835 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3836 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3837
3838 EXPECT_EQ(
3839 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3840 kDefaultFramerate);
3841
3842 // Trigger reconfigure encoder (without resetting the entire instance).
3843 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003844 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003845 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3846 video_encoder_config.number_of_streams = 1;
3847 video_encoder_config.video_stream_factory =
3848 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3849 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003850 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003851 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3852
3853 // Detector should be updated with fps limit from codec config.
3854 EXPECT_EQ(
3855 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3856 kFramerate);
3857
3858 // Trigger overuse, max framerate should be reduced.
3859 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3860 stats.input_frame_rate = kFramerate;
3861 stats_proxy_->SetMockStats(stats);
3862 video_stream_encoder_->TriggerCpuOveruse();
3863 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3864 int adapted_framerate =
3865 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3866 EXPECT_LT(adapted_framerate, kFramerate);
3867
3868 // Trigger underuse, max framerate should go back to codec configured fps.
3869 // Set extra low fps, to make sure it's actually reset, not just incremented.
3870 stats = stats_proxy_->GetStats();
3871 stats.input_frame_rate = adapted_framerate / 2;
3872 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003873 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003874 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3875 EXPECT_EQ(
3876 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3877 kFramerate);
3878
3879 video_stream_encoder_->Stop();
3880}
3881
3882TEST_F(VideoStreamEncoderTest,
3883 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3884 const int kFrameWidth = 1280;
3885 const int kFrameHeight = 720;
3886 const int kLowFramerate = 15;
3887 const int kHighFramerate = 25;
3888
Henrik Boström381d1092020-05-12 18:49:07 +02003889 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003890 DataRate::BitsPerSec(kTargetBitrateBps),
3891 DataRate::BitsPerSec(kTargetBitrateBps),
3892 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003893 test::FrameForwarder source;
3894 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003895 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003896
3897 // Trigger initial configuration.
3898 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003899 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003900 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3901 video_encoder_config.number_of_streams = 1;
3902 video_encoder_config.video_stream_factory =
3903 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3904 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3905 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003906 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003907 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3908
3909 EXPECT_EQ(
3910 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3911 kLowFramerate);
3912
3913 // Trigger overuse, max framerate should be reduced.
3914 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3915 stats.input_frame_rate = kLowFramerate;
3916 stats_proxy_->SetMockStats(stats);
3917 video_stream_encoder_->TriggerCpuOveruse();
3918 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3919 int adapted_framerate =
3920 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3921 EXPECT_LT(adapted_framerate, kLowFramerate);
3922
3923 // Reconfigure the encoder with a new (higher max framerate), max fps should
3924 // still respect the adaptation.
3925 video_encoder_config.video_stream_factory =
3926 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3927 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3928 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003929 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003930 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3931
3932 EXPECT_EQ(
3933 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3934 adapted_framerate);
3935
3936 // Trigger underuse, max framerate should go back to codec configured fps.
3937 stats = stats_proxy_->GetStats();
3938 stats.input_frame_rate = adapted_framerate;
3939 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003940 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003941 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3942 EXPECT_EQ(
3943 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3944 kHighFramerate);
3945
3946 video_stream_encoder_->Stop();
3947}
3948
mflodmancc3d4422017-08-03 08:27:51 -07003949TEST_F(VideoStreamEncoderTest,
3950 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003951 const int kFrameWidth = 1280;
3952 const int kFrameHeight = 720;
3953 const int kFramerate = 24;
3954
Henrik Boström381d1092020-05-12 18:49:07 +02003955 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003956 DataRate::BitsPerSec(kTargetBitrateBps),
3957 DataRate::BitsPerSec(kTargetBitrateBps),
3958 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003959 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003960 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003961 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003962
3963 // Trigger initial configuration.
3964 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003965 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003966 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3967 video_encoder_config.number_of_streams = 1;
3968 video_encoder_config.video_stream_factory =
3969 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3970 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003971 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003972 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003973 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003974
Niels Möller7dc26b72017-12-06 10:27:48 +01003975 EXPECT_EQ(
3976 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3977 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003978
3979 // Trigger overuse, max framerate should be reduced.
3980 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3981 stats.input_frame_rate = kFramerate;
3982 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003983 video_stream_encoder_->TriggerCpuOveruse();
3984 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003985 int adapted_framerate =
3986 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003987 EXPECT_LT(adapted_framerate, kFramerate);
3988
3989 // Change degradation preference to not enable framerate scaling. Target
3990 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02003991 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003992 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01003993 EXPECT_EQ(
3994 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3995 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003996
mflodmancc3d4422017-08-03 08:27:51 -07003997 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003998}
3999
mflodmancc3d4422017-08-03 08:27:51 -07004000TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004001 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004003 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4004 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4005 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004006 const int kWidth = 640;
4007 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004008
asaperssonfab67072017-04-04 05:51:49 -07004009 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004010
4011 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004012 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004013
4014 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004015 EXPECT_TRUE_WAIT(
4016 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004017
sprangc5d62e22017-04-02 23:53:04 -07004018 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004019
asaperssonfab67072017-04-04 05:51:49 -07004020 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004021 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004022 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004023
4024 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004025 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004026
Henrik Boström2671dac2020-05-19 16:29:09 +02004027 EXPECT_TRUE_WAIT(
4028 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004029
mflodmancc3d4422017-08-03 08:27:51 -07004030 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004031}
4032
mflodmancc3d4422017-08-03 08:27:51 -07004033TEST_F(VideoStreamEncoderTest,
4034 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004035 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004037 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4038 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4039 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004040 const int kWidth = 640;
4041 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004042
4043 // We expect the n initial frames to get dropped.
4044 int i;
4045 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004046 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004047 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004048 }
4049 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004050 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004051 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004052
4053 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004054 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004055
mflodmancc3d4422017-08-03 08:27:51 -07004056 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004057}
4058
mflodmancc3d4422017-08-03 08:27:51 -07004059TEST_F(VideoStreamEncoderTest,
4060 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004061 const int kWidth = 640;
4062 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004064 DataRate::BitsPerSec(kLowTargetBitrateBps),
4065 DataRate::BitsPerSec(kLowTargetBitrateBps),
4066 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004067
4068 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004069 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004070 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004071
asaperssonfab67072017-04-04 05:51:49 -07004072 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004073 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004074 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004075
mflodmancc3d4422017-08-03 08:27:51 -07004076 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004077}
4078
mflodmancc3d4422017-08-03 08:27:51 -07004079TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004080 const int kWidth = 640;
4081 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004082 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004083
4084 VideoEncoderConfig video_encoder_config;
4085 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4086 // Make format different, to force recreation of encoder.
4087 video_encoder_config.video_format.parameters["foo"] = "foo";
4088 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004089 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004091 DataRate::BitsPerSec(kLowTargetBitrateBps),
4092 DataRate::BitsPerSec(kLowTargetBitrateBps),
4093 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004094
kthelgasonb83797b2017-02-14 11:57:25 -08004095 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004096 video_stream_encoder_->SetSource(&video_source_,
4097 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004098
asaperssonfab67072017-04-04 05:51:49 -07004099 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004100 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004101 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004102
mflodmancc3d4422017-08-03 08:27:51 -07004103 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004104 fake_encoder_.SetQualityScaling(true);
4105}
4106
Åsa Persson139f4dc2019-08-02 09:29:58 +02004107TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4108 webrtc::test::ScopedFieldTrials field_trials(
4109 "WebRTC-Video-QualityScalerSettings/"
4110 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4111 // Reset encoder for field trials to take effect.
4112 ConfigureEncoder(video_encoder_config_.Copy());
4113 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4114 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4115 const int kWidth = 640;
4116 const int kHeight = 360;
4117
Henrik Boström381d1092020-05-12 18:49:07 +02004118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004119 DataRate::BitsPerSec(kTargetBitrateBps),
4120 DataRate::BitsPerSec(kTargetBitrateBps),
4121 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004122 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4123 // Frame should not be dropped.
4124 WaitForEncodedFrame(1);
4125
Henrik Boström381d1092020-05-12 18:49:07 +02004126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004127 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4128 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4129 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004130 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4131 // Frame should not be dropped.
4132 WaitForEncodedFrame(2);
4133
Henrik Boström381d1092020-05-12 18:49:07 +02004134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004135 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4136 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4137 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004138 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4139 // Expect to drop this frame, the wait should time out.
4140 ExpectDroppedFrame();
4141
4142 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004143 EXPECT_TRUE_WAIT(
4144 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004145 video_stream_encoder_->Stop();
4146}
4147
Åsa Perssone644a032019-11-08 15:56:00 +01004148TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4149 webrtc::test::ScopedFieldTrials field_trials(
4150 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4151
4152 // Reset encoder for field trials to take effect.
4153 VideoEncoderConfig config = video_encoder_config_.Copy();
4154 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004155 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004156 ConfigureEncoder(std::move(config));
4157 fake_encoder_.SetQp(kQpLow);
4158
4159 // Enable MAINTAIN_FRAMERATE preference.
4160 AdaptingFrameForwarder source;
4161 source.set_adaptation_enabled(true);
4162 video_stream_encoder_->SetSource(&source,
4163 DegradationPreference::MAINTAIN_FRAMERATE);
4164
4165 // Start at low bitrate.
4166 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4168 DataRate::BitsPerSec(kLowBitrateBps),
4169 DataRate::BitsPerSec(kLowBitrateBps),
4170 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004171
4172 // Expect first frame to be dropped and resolution to be limited.
4173 const int kWidth = 1280;
4174 const int kHeight = 720;
4175 const int64_t kFrameIntervalMs = 100;
4176 int64_t timestamp_ms = kFrameIntervalMs;
4177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4178 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004179 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4180 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004181
4182 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4184 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004185
4186 // Insert frames and advance |min_duration_ms|.
4187 for (size_t i = 1; i <= 10; i++) {
4188 timestamp_ms += kFrameIntervalMs;
4189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4190 WaitForEncodedFrame(timestamp_ms);
4191 }
4192 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4193 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4194
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004195 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004196
4197 // Insert frame should trigger high BW and release quality limitation.
4198 timestamp_ms += kFrameIntervalMs;
4199 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4200 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004201 // The ramp-up code involves the adaptation queue, give it time to execute.
4202 // TODO(hbos): Can we await an appropriate event instead?
4203 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004204 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004205
4206 // Frame should not be adapted.
4207 timestamp_ms += kFrameIntervalMs;
4208 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4209 WaitForEncodedFrame(kWidth, kHeight);
4210 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4211
4212 video_stream_encoder_->Stop();
4213}
4214
mflodmancc3d4422017-08-03 08:27:51 -07004215TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004216 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4217 const int kTooSmallWidth = 10;
4218 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004220 DataRate::BitsPerSec(kTargetBitrateBps),
4221 DataRate::BitsPerSec(kTargetBitrateBps),
4222 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004223
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004224 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004225 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004226 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004227 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004228 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004229 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4230
4231 // Trigger adapt down, too small frame, expect no change.
4232 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004233 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004234 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004235 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004236 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4237 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4238
mflodmancc3d4422017-08-03 08:27:51 -07004239 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004240}
4241
mflodmancc3d4422017-08-03 08:27:51 -07004242TEST_F(VideoStreamEncoderTest,
4243 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004244 const int kTooSmallWidth = 10;
4245 const int kTooSmallHeight = 10;
4246 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004247 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004248 DataRate::BitsPerSec(kTargetBitrateBps),
4249 DataRate::BitsPerSec(kTargetBitrateBps),
4250 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004251
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004252 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004253 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004254 video_stream_encoder_->SetSource(&source,
4255 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004256 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4258 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4259
4260 // Trigger adapt down, expect limited framerate.
4261 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004262 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004263 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004264 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004265 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4266 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4267 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4268
4269 // Trigger adapt down, too small frame, expect no change.
4270 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004271 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004272 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004273 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004274 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4275 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4276 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4277
mflodmancc3d4422017-08-03 08:27:51 -07004278 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004279}
4280
mflodmancc3d4422017-08-03 08:27:51 -07004281TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004282 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004284 DataRate::BitsPerSec(kTargetBitrateBps),
4285 DataRate::BitsPerSec(kTargetBitrateBps),
4286 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004287 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004288 const int kFrameWidth = 1280;
4289 const int kFrameHeight = 720;
4290 video_source_.IncomingCapturedFrame(
4291 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004292 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004293 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004294}
4295
sprangb1ca0732017-02-01 08:38:12 -08004296// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004297TEST_F(VideoStreamEncoderTest,
4298 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004299 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004300 DataRate::BitsPerSec(kTargetBitrateBps),
4301 DataRate::BitsPerSec(kTargetBitrateBps),
4302 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004303
4304 const int kFrameWidth = 1280;
4305 const int kFrameHeight = 720;
4306 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004307 // requested by
4308 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004309 video_source_.set_adaptation_enabled(true);
4310
4311 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004312 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004313 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004314
4315 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004316 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004317 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004318 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004319 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004320
asaperssonfab67072017-04-04 05:51:49 -07004321 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004322 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004323 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004324 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004325 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004326
mflodmancc3d4422017-08-03 08:27:51 -07004327 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004328}
sprangfe627f32017-03-29 08:24:59 -07004329
mflodmancc3d4422017-08-03 08:27:51 -07004330TEST_F(VideoStreamEncoderTest,
4331 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004332 const int kFrameWidth = 1280;
4333 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004334
Henrik Boström381d1092020-05-12 18:49:07 +02004335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004336 DataRate::BitsPerSec(kTargetBitrateBps),
4337 DataRate::BitsPerSec(kTargetBitrateBps),
4338 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004339 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004340 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004341 video_source_.set_adaptation_enabled(true);
4342
sprang4847ae62017-06-27 07:06:52 -07004343 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004344
4345 video_source_.IncomingCapturedFrame(
4346 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004347 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004348
4349 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004350 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004351
4352 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004353 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004354 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004355 video_source_.IncomingCapturedFrame(
4356 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004357 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004358 }
4359
4360 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004361 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004362 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004363 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004364 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004365 video_source_.IncomingCapturedFrame(
4366 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004367 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004368 ++num_frames_dropped;
4369 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004370 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004371 }
4372 }
4373
sprang4847ae62017-06-27 07:06:52 -07004374 // Add some slack to account for frames dropped by the frame dropper.
4375 const int kErrorMargin = 1;
4376 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004377 kErrorMargin);
4378
4379 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004380 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004381 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004382 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004383 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004384 video_source_.IncomingCapturedFrame(
4385 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004386 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004387 ++num_frames_dropped;
4388 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004389 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004390 }
4391 }
sprang4847ae62017-06-27 07:06:52 -07004392 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004393 kErrorMargin);
4394
4395 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004396 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004397 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004398 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004399 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004400 video_source_.IncomingCapturedFrame(
4401 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004402 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004403 ++num_frames_dropped;
4404 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004405 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004406 }
4407 }
sprang4847ae62017-06-27 07:06:52 -07004408 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004409 kErrorMargin);
4410
4411 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004412 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004413 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004414 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004415 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004416 video_source_.IncomingCapturedFrame(
4417 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004418 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004419 ++num_frames_dropped;
4420 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004421 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004422 }
4423 }
4424 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4425
mflodmancc3d4422017-08-03 08:27:51 -07004426 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004427}
4428
mflodmancc3d4422017-08-03 08:27:51 -07004429TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004430 const int kFramerateFps = 5;
4431 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004432 const int kFrameWidth = 1280;
4433 const int kFrameHeight = 720;
4434
sprang4847ae62017-06-27 07:06:52 -07004435 // Reconfigure encoder with two temporal layers and screensharing, which will
4436 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004437 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004438
Henrik Boström381d1092020-05-12 18:49:07 +02004439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004440 DataRate::BitsPerSec(kTargetBitrateBps),
4441 DataRate::BitsPerSec(kTargetBitrateBps),
4442 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004443 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004444 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004445 video_source_.set_adaptation_enabled(true);
4446
sprang4847ae62017-06-27 07:06:52 -07004447 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004448
4449 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004450 rtc::VideoSinkWants last_wants;
4451 do {
4452 last_wants = video_source_.sink_wants();
4453
sprangc5d62e22017-04-02 23:53:04 -07004454 // Insert frames to get a new fps estimate...
4455 for (int j = 0; j < kFramerateFps; ++j) {
4456 video_source_.IncomingCapturedFrame(
4457 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004458 if (video_source_.last_sent_width()) {
4459 sink_.WaitForEncodedFrame(timestamp_ms);
4460 }
sprangc5d62e22017-04-02 23:53:04 -07004461 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004462 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004463 }
4464 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004465 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004466 } while (video_source_.sink_wants().max_framerate_fps <
4467 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004468
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004469 EXPECT_THAT(video_source_.sink_wants(),
4470 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004471
mflodmancc3d4422017-08-03 08:27:51 -07004472 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004473}
asaperssonf7e294d2017-06-13 23:25:22 -07004474
mflodmancc3d4422017-08-03 08:27:51 -07004475TEST_F(VideoStreamEncoderTest,
4476 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004477 const int kWidth = 1280;
4478 const int kHeight = 720;
4479 const int64_t kFrameIntervalMs = 150;
4480 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004482 DataRate::BitsPerSec(kTargetBitrateBps),
4483 DataRate::BitsPerSec(kTargetBitrateBps),
4484 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004485
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004486 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004487 AdaptingFrameForwarder source;
4488 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004489 video_stream_encoder_->SetSource(&source,
4490 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004491 timestamp_ms += kFrameIntervalMs;
4492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004493 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004494 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004495 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4496 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4497 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4498
4499 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004500 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004501 timestamp_ms += kFrameIntervalMs;
4502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004503 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004504 EXPECT_THAT(source.sink_wants(),
4505 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004506 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4507 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4508 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4509
4510 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004511 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004512 timestamp_ms += kFrameIntervalMs;
4513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004514 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004515 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4517 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4518 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4519
4520 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004521 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004522 timestamp_ms += kFrameIntervalMs;
4523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004524 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004525 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004526 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4528 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4529
4530 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004531 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004532 timestamp_ms += kFrameIntervalMs;
4533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004534 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004535 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4538 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4539
4540 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004542 timestamp_ms += kFrameIntervalMs;
4543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004544 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004545 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4548 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4549
4550 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004551 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004552 timestamp_ms += kFrameIntervalMs;
4553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004555 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4558 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4559
4560 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004562 timestamp_ms += kFrameIntervalMs;
4563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004565 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004566 rtc::VideoSinkWants last_wants = source.sink_wants();
4567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4569 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4570
4571 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004572 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004573 timestamp_ms += kFrameIntervalMs;
4574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004575 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004576 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4579 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4580
Evan Shrubsole64469032020-06-11 10:45:29 +02004581 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004582 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004583 timestamp_ms += kFrameIntervalMs;
4584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004585 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004586 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4589 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4590
4591 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004592 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004593 timestamp_ms += kFrameIntervalMs;
4594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004596 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4599 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4600
4601 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004602 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004603 timestamp_ms += kFrameIntervalMs;
4604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004605 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004606 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4609 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4610
4611 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004612 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004613 timestamp_ms += kFrameIntervalMs;
4614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004616 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4619 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4620
4621 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004622 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004623 timestamp_ms += kFrameIntervalMs;
4624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004625 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004626 EXPECT_THAT(source.sink_wants(), FpsMax());
4627 EXPECT_EQ(source.sink_wants().max_pixel_count,
4628 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4630 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4631 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4632
4633 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004634 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004635 timestamp_ms += kFrameIntervalMs;
4636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004637 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004638 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004639 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4640 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4641 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4642
Åsa Persson30ab0152019-08-27 12:22:33 +02004643 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004644 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004645 timestamp_ms += kFrameIntervalMs;
4646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004647 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004648 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004649 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4652 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4653
4654 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004655 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004656 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004657 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4658
mflodmancc3d4422017-08-03 08:27:51 -07004659 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004660}
4661
mflodmancc3d4422017-08-03 08:27:51 -07004662TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004663 const int kWidth = 1280;
4664 const int kHeight = 720;
4665 const int64_t kFrameIntervalMs = 150;
4666 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004668 DataRate::BitsPerSec(kTargetBitrateBps),
4669 DataRate::BitsPerSec(kTargetBitrateBps),
4670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004671
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004672 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004673 AdaptingFrameForwarder source;
4674 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004675 video_stream_encoder_->SetSource(&source,
4676 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004677 timestamp_ms += kFrameIntervalMs;
4678 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004679 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004680 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004681 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4682 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4683 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4685 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4686 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4687
4688 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004689 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004690 timestamp_ms += kFrameIntervalMs;
4691 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004692 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004693 EXPECT_THAT(source.sink_wants(),
4694 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004695 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4696 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4697 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4698 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4699 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4700 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4701
4702 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004703 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004704 timestamp_ms += kFrameIntervalMs;
4705 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004706 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004707 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4710 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4711 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4712 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4713 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4714
4715 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004716 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004717 timestamp_ms += kFrameIntervalMs;
4718 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004719 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004720 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02004721 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07004722 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4723 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4724 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4725 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4726 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4727
Evan Shrubsole64469032020-06-11 10:45:29 +02004728 // Trigger cpu adapt up, expect no change since QP is most limited.
4729 {
4730 // Store current sink wants since we expect no change and if there is no
4731 // change then last_wants() is not updated.
4732 auto previous_sink_wants = source.sink_wants();
4733 video_stream_encoder_->TriggerCpuUnderuse();
4734 timestamp_ms += kFrameIntervalMs;
4735 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4736 WaitForEncodedFrame(timestamp_ms);
4737 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4738 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4739 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4740 }
4741
4742 // Trigger quality adapt up, expect increased fps (640x360@30fps).
4743 video_stream_encoder_->TriggerQualityHigh();
4744 timestamp_ms += kFrameIntervalMs;
4745 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4746 WaitForEncodedFrame(timestamp_ms);
4747 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
4748 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4749 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4750 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4752 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4753 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4754
4755 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4756 // expect increased resolution (960x540@30fps).
4757 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004758 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004759 timestamp_ms += kFrameIntervalMs;
4760 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004761 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02004762 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004763 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4764 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4765 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4766 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4767 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004768 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004769
Evan Shrubsole64469032020-06-11 10:45:29 +02004770 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4771 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004772 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004773 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004774 timestamp_ms += kFrameIntervalMs;
4775 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004776 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004777 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004778 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004779 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4780 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4781 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4782 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4783 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004784 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004785
4786 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004787 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004788 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004789 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004790 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004791
mflodmancc3d4422017-08-03 08:27:51 -07004792 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004793}
4794
mflodmancc3d4422017-08-03 08:27:51 -07004795TEST_F(VideoStreamEncoderTest,
4796 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004797 const int kWidth = 640;
4798 const int kHeight = 360;
4799 const int kFpsLimit = 15;
4800 const int64_t kFrameIntervalMs = 150;
4801 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004803 DataRate::BitsPerSec(kTargetBitrateBps),
4804 DataRate::BitsPerSec(kTargetBitrateBps),
4805 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004806
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004807 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004808 AdaptingFrameForwarder source;
4809 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004810 video_stream_encoder_->SetSource(&source,
4811 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004812 timestamp_ms += kFrameIntervalMs;
4813 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004814 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4818 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4819 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4820 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4821 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4822
4823 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004824 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004825 timestamp_ms += kFrameIntervalMs;
4826 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004827 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004828 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4830 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4831 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4832 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4833 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4834 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4835
4836 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004837 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004838 timestamp_ms += kFrameIntervalMs;
4839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004840 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004841 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004842 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004843 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07004844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4845 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4846 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4847 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4848
Evan Shrubsole64469032020-06-11 10:45:29 +02004849 // Trigger cpu adapt up, expect no change because quality is most limited.
4850 {
4851 auto previous_sink_wants = source.sink_wants();
4852 // Store current sink wants since we expect no change ind if there is no
4853 // change then last__wants() is not updated.
4854 video_stream_encoder_->TriggerCpuUnderuse();
4855 timestamp_ms += kFrameIntervalMs;
4856 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4857 WaitForEncodedFrame(timestamp_ms);
4858 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4859 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4861 }
4862
4863 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
4864 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004865 timestamp_ms += kFrameIntervalMs;
4866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004867 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004868 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004869 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4870 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004872 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4874 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004875
Evan Shrubsole64469032020-06-11 10:45:29 +02004876 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004877 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02004878 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004879 timestamp_ms += kFrameIntervalMs;
4880 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004881 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004882 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4884 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4885 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4886 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4887 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004888 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004889
4890 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004891 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004892 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004893 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004894 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004895
mflodmancc3d4422017-08-03 08:27:51 -07004896 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004897}
4898
mflodmancc3d4422017-08-03 08:27:51 -07004899TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004900 const int kFrameWidth = 1920;
4901 const int kFrameHeight = 1080;
4902 // 3/4 of 1920.
4903 const int kAdaptedFrameWidth = 1440;
4904 // 3/4 of 1080 rounded down to multiple of 4.
4905 const int kAdaptedFrameHeight = 808;
4906 const int kFramerate = 24;
4907
Henrik Boström381d1092020-05-12 18:49:07 +02004908 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004909 DataRate::BitsPerSec(kTargetBitrateBps),
4910 DataRate::BitsPerSec(kTargetBitrateBps),
4911 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004912 // Trigger reconfigure encoder (without resetting the entire instance).
4913 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004914 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004915 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4916 video_encoder_config.number_of_streams = 1;
4917 video_encoder_config.video_stream_factory =
4918 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004919 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004920 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004921 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004922
4923 video_source_.set_adaptation_enabled(true);
4924
4925 video_source_.IncomingCapturedFrame(
4926 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004927 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004928
4929 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004930 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004931 video_source_.IncomingCapturedFrame(
4932 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004933 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004934
mflodmancc3d4422017-08-03 08:27:51 -07004935 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004936}
4937
mflodmancc3d4422017-08-03 08:27:51 -07004938TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004939 const int kFrameWidth = 1280;
4940 const int kFrameHeight = 720;
4941 const int kLowFps = 2;
4942 const int kHighFps = 30;
4943
Henrik Boström381d1092020-05-12 18:49:07 +02004944 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004945 DataRate::BitsPerSec(kTargetBitrateBps),
4946 DataRate::BitsPerSec(kTargetBitrateBps),
4947 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004948
4949 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4950 max_framerate_ = kLowFps;
4951
4952 // Insert 2 seconds of 2fps video.
4953 for (int i = 0; i < kLowFps * 2; ++i) {
4954 video_source_.IncomingCapturedFrame(
4955 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4956 WaitForEncodedFrame(timestamp_ms);
4957 timestamp_ms += 1000 / kLowFps;
4958 }
4959
4960 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004962 DataRate::BitsPerSec(kTargetBitrateBps),
4963 DataRate::BitsPerSec(kTargetBitrateBps),
4964 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004965 video_source_.IncomingCapturedFrame(
4966 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4967 WaitForEncodedFrame(timestamp_ms);
4968 timestamp_ms += 1000 / kLowFps;
4969
4970 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
4971
4972 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02004973 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07004974 const int kFrameIntervalMs = 1000 / kHighFps;
4975 max_framerate_ = kHighFps;
4976 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
4977 video_source_.IncomingCapturedFrame(
4978 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4979 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
4980 // be dropped if the encoder hans't been updated with the new higher target
4981 // framerate yet, causing it to overshoot the target bitrate and then
4982 // suffering the wrath of the media optimizer.
4983 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
4984 timestamp_ms += kFrameIntervalMs;
4985 }
4986
4987 // Don expect correct measurement just yet, but it should be higher than
4988 // before.
4989 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
4990
mflodmancc3d4422017-08-03 08:27:51 -07004991 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07004992}
4993
mflodmancc3d4422017-08-03 08:27:51 -07004994TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07004995 const int kFrameWidth = 1280;
4996 const int kFrameHeight = 720;
4997 const int kTargetBitrateBps = 1000000;
4998
4999 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02005000 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02005001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005002 DataRate::BitsPerSec(kTargetBitrateBps),
5003 DataRate::BitsPerSec(kTargetBitrateBps),
5004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005005 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005006
5007 // Insert a first video frame, causes another bitrate update.
5008 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5009 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5010 video_source_.IncomingCapturedFrame(
5011 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5012 WaitForEncodedFrame(timestamp_ms);
5013
5014 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005015 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5016 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5017 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005018
5019 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005020 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005021 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005022
5023 // Bitrate observer should not be called.
5024 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5025 video_source_.IncomingCapturedFrame(
5026 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5027 ExpectDroppedFrame();
5028
mflodmancc3d4422017-08-03 08:27:51 -07005029 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005030}
ilnik6b826ef2017-06-16 06:53:48 -07005031
Niels Möller4db138e2018-04-19 09:04:13 +02005032TEST_F(VideoStreamEncoderTest,
5033 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5034 const int kFrameWidth = 1280;
5035 const int kFrameHeight = 720;
5036 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005038 DataRate::BitsPerSec(kTargetBitrateBps),
5039 DataRate::BitsPerSec(kTargetBitrateBps),
5040 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005041 video_source_.IncomingCapturedFrame(
5042 CreateFrame(1, kFrameWidth, kFrameHeight));
5043 WaitForEncodedFrame(1);
5044 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5045 .low_encode_usage_threshold_percent,
5046 default_options.low_encode_usage_threshold_percent);
5047 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5048 .high_encode_usage_threshold_percent,
5049 default_options.high_encode_usage_threshold_percent);
5050 video_stream_encoder_->Stop();
5051}
5052
5053TEST_F(VideoStreamEncoderTest,
5054 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5055 const int kFrameWidth = 1280;
5056 const int kFrameHeight = 720;
5057 CpuOveruseOptions hardware_options;
5058 hardware_options.low_encode_usage_threshold_percent = 150;
5059 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005060 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005061
Henrik Boström381d1092020-05-12 18:49:07 +02005062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005063 DataRate::BitsPerSec(kTargetBitrateBps),
5064 DataRate::BitsPerSec(kTargetBitrateBps),
5065 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005066 video_source_.IncomingCapturedFrame(
5067 CreateFrame(1, kFrameWidth, kFrameHeight));
5068 WaitForEncodedFrame(1);
5069 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5070 .low_encode_usage_threshold_percent,
5071 hardware_options.low_encode_usage_threshold_percent);
5072 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5073 .high_encode_usage_threshold_percent,
5074 hardware_options.high_encode_usage_threshold_percent);
5075 video_stream_encoder_->Stop();
5076}
5077
Niels Möller6bb5ab92019-01-11 11:11:10 +01005078TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5079 const int kFrameWidth = 320;
5080 const int kFrameHeight = 240;
5081 const int kFps = 30;
5082 const int kTargetBitrateBps = 120000;
5083 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5084
Henrik Boström381d1092020-05-12 18:49:07 +02005085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005086 DataRate::BitsPerSec(kTargetBitrateBps),
5087 DataRate::BitsPerSec(kTargetBitrateBps),
5088 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005089
5090 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5091 max_framerate_ = kFps;
5092
5093 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5094 fake_encoder_.SimulateOvershoot(1.0);
5095 int num_dropped = 0;
5096 for (int i = 0; i < kNumFramesInRun; ++i) {
5097 video_source_.IncomingCapturedFrame(
5098 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5099 // Wait up to two frame durations for a frame to arrive.
5100 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5101 ++num_dropped;
5102 }
5103 timestamp_ms += 1000 / kFps;
5104 }
5105
Erik Språnga8d48ab2019-02-08 14:17:40 +01005106 // Framerate should be measured to be near the expected target rate.
5107 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5108
5109 // Frame drops should be within 5% of expected 0%.
5110 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005111
5112 // Make encoder produce frames at double the expected bitrate during 3 seconds
5113 // of video, verify number of drops. Rate needs to be slightly changed in
5114 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005115 double overshoot_factor = 2.0;
5116 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5117 // With bitrate adjuster, when need to overshoot even more to trigger
5118 // frame dropping.
5119 overshoot_factor *= 2;
5120 }
5121 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005122 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005123 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5124 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5125 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005126 num_dropped = 0;
5127 for (int i = 0; i < kNumFramesInRun; ++i) {
5128 video_source_.IncomingCapturedFrame(
5129 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5130 // Wait up to two frame durations for a frame to arrive.
5131 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5132 ++num_dropped;
5133 }
5134 timestamp_ms += 1000 / kFps;
5135 }
5136
Henrik Boström381d1092020-05-12 18:49:07 +02005137 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005138 DataRate::BitsPerSec(kTargetBitrateBps),
5139 DataRate::BitsPerSec(kTargetBitrateBps),
5140 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005141
5142 // Target framerate should be still be near the expected target, despite
5143 // the frame drops.
5144 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5145
5146 // Frame drops should be within 5% of expected 50%.
5147 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005148
5149 video_stream_encoder_->Stop();
5150}
5151
5152TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5153 const int kFrameWidth = 320;
5154 const int kFrameHeight = 240;
5155 const int kActualInputFps = 24;
5156 const int kTargetBitrateBps = 120000;
5157
5158 ASSERT_GT(max_framerate_, kActualInputFps);
5159
5160 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5161 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005163 DataRate::BitsPerSec(kTargetBitrateBps),
5164 DataRate::BitsPerSec(kTargetBitrateBps),
5165 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005166
5167 // Insert 3 seconds of video, with an input fps lower than configured max.
5168 for (int i = 0; i < kActualInputFps * 3; ++i) {
5169 video_source_.IncomingCapturedFrame(
5170 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5171 // Wait up to two frame durations for a frame to arrive.
5172 WaitForEncodedFrame(timestamp_ms);
5173 timestamp_ms += 1000 / kActualInputFps;
5174 }
5175
5176 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5177
5178 video_stream_encoder_->Stop();
5179}
5180
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005181TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5182 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005184 DataRate::BitsPerSec(kTargetBitrateBps),
5185 DataRate::BitsPerSec(kTargetBitrateBps),
5186 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005187
5188 fake_encoder_.BlockNextEncode();
5189 video_source_.IncomingCapturedFrame(
5190 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5191 WaitForEncodedFrame(1);
5192 // On the very first frame full update should be forced.
5193 rect = fake_encoder_.GetLastUpdateRect();
5194 EXPECT_EQ(rect.offset_x, 0);
5195 EXPECT_EQ(rect.offset_y, 0);
5196 EXPECT_EQ(rect.height, codec_height_);
5197 EXPECT_EQ(rect.width, codec_width_);
5198 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5199 // call to ContinueEncode.
5200 video_source_.IncomingCapturedFrame(
5201 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5202 ExpectDroppedFrame();
5203 video_source_.IncomingCapturedFrame(
5204 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5205 ExpectDroppedFrame();
5206 fake_encoder_.ContinueEncode();
5207 WaitForEncodedFrame(3);
5208 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5209 rect = fake_encoder_.GetLastUpdateRect();
5210 EXPECT_EQ(rect.offset_x, 1);
5211 EXPECT_EQ(rect.offset_y, 0);
5212 EXPECT_EQ(rect.width, 10);
5213 EXPECT_EQ(rect.height, 1);
5214
5215 video_source_.IncomingCapturedFrame(
5216 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5217 WaitForEncodedFrame(4);
5218 // Previous frame was encoded, so no accumulation should happen.
5219 rect = fake_encoder_.GetLastUpdateRect();
5220 EXPECT_EQ(rect.offset_x, 0);
5221 EXPECT_EQ(rect.offset_y, 0);
5222 EXPECT_EQ(rect.width, 1);
5223 EXPECT_EQ(rect.height, 1);
5224
5225 video_stream_encoder_->Stop();
5226}
5227
Erik Språngd7329ca2019-02-21 21:19:53 +01005228TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005230 DataRate::BitsPerSec(kTargetBitrateBps),
5231 DataRate::BitsPerSec(kTargetBitrateBps),
5232 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005233
5234 // First frame is always keyframe.
5235 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5236 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005237 EXPECT_THAT(
5238 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005239 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005240
5241 // Insert delta frame.
5242 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5243 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005244 EXPECT_THAT(
5245 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005246 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005247
5248 // Request next frame be a key-frame.
5249 video_stream_encoder_->SendKeyFrame();
5250 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5251 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005252 EXPECT_THAT(
5253 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005254 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005255
5256 video_stream_encoder_->Stop();
5257}
5258
5259TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5260 // Setup simulcast with three streams.
5261 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005263 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5264 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5265 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005266 // Wait for all three layers before triggering event.
5267 sink_.SetNumExpectedLayers(3);
5268
5269 // First frame is always keyframe.
5270 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5271 WaitForEncodedFrame(1);
5272 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005273 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5274 VideoFrameType::kVideoFrameKey,
5275 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005276
5277 // Insert delta frame.
5278 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5279 WaitForEncodedFrame(2);
5280 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005281 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5282 VideoFrameType::kVideoFrameDelta,
5283 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005284
5285 // Request next frame be a key-frame.
5286 // Only first stream is configured to produce key-frame.
5287 video_stream_encoder_->SendKeyFrame();
5288 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5289 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005290
5291 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5292 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005293 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005294 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005295 VideoFrameType::kVideoFrameKey,
5296 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005297
5298 video_stream_encoder_->Stop();
5299}
5300
5301TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5302 // Configure internal source factory and setup test again.
5303 encoder_factory_.SetHasInternalSource(true);
5304 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005306 DataRate::BitsPerSec(kTargetBitrateBps),
5307 DataRate::BitsPerSec(kTargetBitrateBps),
5308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005309
5310 // Call encoder directly, simulating internal source where encoded frame
5311 // callback in VideoStreamEncoder is called despite no OnFrame().
5312 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5313 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005314 EXPECT_THAT(
5315 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005316 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005317
Niels Möller8f7ce222019-03-21 15:43:58 +01005318 const std::vector<VideoFrameType> kDeltaFrame = {
5319 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005320 // Need to set timestamp manually since manually for injected frame.
5321 VideoFrame frame = CreateFrame(101, nullptr);
5322 frame.set_timestamp(101);
5323 fake_encoder_.InjectFrame(frame, false);
5324 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005325 EXPECT_THAT(
5326 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005327 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005328
5329 // Request key-frame. The forces a dummy frame down into the encoder.
5330 fake_encoder_.ExpectNullFrame();
5331 video_stream_encoder_->SendKeyFrame();
5332 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005333 EXPECT_THAT(
5334 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005335 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005336
5337 video_stream_encoder_->Stop();
5338}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005339
5340TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5341 // Configure internal source factory and setup test again.
5342 encoder_factory_.SetHasInternalSource(true);
5343 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005344 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005345 DataRate::BitsPerSec(kTargetBitrateBps),
5346 DataRate::BitsPerSec(kTargetBitrateBps),
5347 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005348
5349 int64_t timestamp = 1;
5350 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005351 image.SetEncodedData(
5352 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005353 image.capture_time_ms_ = ++timestamp;
5354 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5355 const int64_t kEncodeFinishDelayMs = 10;
5356 image.timing_.encode_start_ms = timestamp;
5357 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5358 fake_encoder_.InjectEncodedImage(image);
5359 // Wait for frame without incrementing clock.
5360 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5361 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5362 // capture timestamp should be kEncodeFinishDelayMs in the past.
5363 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5364 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5365 kEncodeFinishDelayMs);
5366
5367 video_stream_encoder_->Stop();
5368}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005369
5370TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005371 // SPS contains VUI with restrictions on the maximum number of reordered
5372 // pictures, there is no need to rewrite the bitstream to enable faster
5373 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005374 ResetEncoder("H264", 1, 1, 1, false);
5375
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5377 DataRate::BitsPerSec(kTargetBitrateBps),
5378 DataRate::BitsPerSec(kTargetBitrateBps),
5379 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5380 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005381
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005382 fake_encoder_.SetEncodedImageData(
5383 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005384
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005385 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5386 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005387
5388 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5389 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005390
5391 video_stream_encoder_->Stop();
5392}
5393
5394TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005395 // SPS does not contain VUI, the bitstream is will be rewritten with added
5396 // VUI with restrictions on the maximum number of reordered pictures to
5397 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005398 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5399 0x00, 0x00, 0x03, 0x03, 0xF4,
5400 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005401 ResetEncoder("H264", 1, 1, 1, false);
5402
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005403 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5404 DataRate::BitsPerSec(kTargetBitrateBps),
5405 DataRate::BitsPerSec(kTargetBitrateBps),
5406 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5407 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005408
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005409 fake_encoder_.SetEncodedImageData(
5410 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005411
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005412 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5413 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005414
5415 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5416 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005417
5418 video_stream_encoder_->Stop();
5419}
5420
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005421TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5422 const int kFrameWidth = 1280;
5423 const int kFrameHeight = 720;
5424 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5425
Henrik Boström381d1092020-05-12 18:49:07 +02005426 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005427 DataRate::BitsPerSec(kTargetBitrateBps),
5428 DataRate::BitsPerSec(kTargetBitrateBps),
5429 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5431
5432 // Insert a first video frame. It should be dropped because of downscale in
5433 // resolution.
5434 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5435 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5436 frame.set_rotation(kVideoRotation_270);
5437 video_source_.IncomingCapturedFrame(frame);
5438
5439 ExpectDroppedFrame();
5440
5441 // Second frame is downscaled.
5442 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5443 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5444 frame.set_rotation(kVideoRotation_90);
5445 video_source_.IncomingCapturedFrame(frame);
5446
5447 WaitForEncodedFrame(timestamp_ms);
5448 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5449
5450 // Insert another frame, also downscaled.
5451 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5452 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5453 frame.set_rotation(kVideoRotation_180);
5454 video_source_.IncomingCapturedFrame(frame);
5455
5456 WaitForEncodedFrame(timestamp_ms);
5457 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5458
5459 video_stream_encoder_->Stop();
5460}
5461
Erik Språng5056af02019-09-02 15:53:11 +02005462TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5463 const int kFrameWidth = 320;
5464 const int kFrameHeight = 180;
5465
5466 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005468 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5469 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5470 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005471 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005472 /*rtt_ms=*/0,
5473 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005474
5475 // Insert a first video frame so that encoder gets configured.
5476 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5477 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5478 frame.set_rotation(kVideoRotation_270);
5479 video_source_.IncomingCapturedFrame(frame);
5480 WaitForEncodedFrame(timestamp_ms);
5481
5482 // Set a target rate below the minimum allowed by the codec settings.
5483 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005484 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5485 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005487 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005488 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005489 /*link_allocation=*/target_rate,
5490 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005491 /*rtt_ms=*/0,
5492 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005493 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5494
5495 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5496 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5497 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005498 DataRate allocation_sum =
5499 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005500 EXPECT_EQ(min_rate, allocation_sum);
5501 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5502
5503 video_stream_encoder_->Stop();
5504}
5505
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005506TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005508 DataRate::BitsPerSec(kTargetBitrateBps),
5509 DataRate::BitsPerSec(kTargetBitrateBps),
5510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005511 // Capture a frame and wait for it to synchronize with the encoder thread.
5512 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5513 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5514 WaitForEncodedFrame(1);
5515
5516 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5517 ASSERT_TRUE(prev_rate_settings.has_value());
5518 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5519 kDefaultFramerate);
5520
5521 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5522 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5523 timestamp_ms += 1000 / kDefaultFramerate;
5524 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5525 WaitForEncodedFrame(timestamp_ms);
5526 }
5527 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5528 kDefaultFramerate);
5529 // Capture larger frame to trigger a reconfigure.
5530 codec_height_ *= 2;
5531 codec_width_ *= 2;
5532 timestamp_ms += 1000 / kDefaultFramerate;
5533 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5534 WaitForEncodedFrame(timestamp_ms);
5535
5536 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5537 auto current_rate_settings =
5538 fake_encoder_.GetAndResetLastRateControlSettings();
5539 // Ensure we have actually reconfigured twice
5540 // The rate settings should have been set again even though
5541 // they haven't changed.
5542 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005543 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005544
5545 video_stream_encoder_->Stop();
5546}
5547
philipeld9cc8c02019-09-16 14:53:40 +02005548struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005549 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5550 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5551 MOCK_METHOD(void,
5552 RequestEncoderSwitch,
5553 (const webrtc::SdpVideoFormat& format),
5554 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005555};
5556
5557TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5558 constexpr int kDontCare = 100;
5559
5560 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5561 video_send_config_.encoder_settings.encoder_switch_request_callback =
5562 &switch_callback;
5563 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5564 encoder_config.codec_type = kVideoCodecVP8;
5565 webrtc::test::ScopedFieldTrials field_trial(
5566 "WebRTC-NetworkCondition-EncoderSwitch/"
5567 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5568 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5569
5570 // Reset encoder for new configuration to take effect.
5571 ConfigureEncoder(std::move(encoder_config));
5572
5573 // Send one frame to trigger ReconfigureEncoder.
5574 video_source_.IncomingCapturedFrame(
5575 CreateFrame(kDontCare, kDontCare, kDontCare));
5576
5577 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005578 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5579 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005580 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005581 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005582
Henrik Boström381d1092020-05-12 18:49:07 +02005583 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005584 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5585 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5586 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005587 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005588 /*rtt_ms=*/0,
5589 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005590
5591 video_stream_encoder_->Stop();
5592}
5593
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005594TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5595 constexpr int kDontCare = 100;
5596
5597 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5598 video_send_config_.encoder_settings.encoder_switch_request_callback =
5599 &switch_callback;
5600 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5601 encoder_config.codec_type = kVideoCodecVP8;
5602 webrtc::test::ScopedFieldTrials field_trial(
5603 "WebRTC-NetworkCondition-EncoderSwitch/"
5604 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5605 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5606
5607 // Reset encoder for new configuration to take effect.
5608 ConfigureEncoder(std::move(encoder_config));
5609
5610 // Send one frame to trigger ReconfigureEncoder.
5611 video_source_.IncomingCapturedFrame(
5612 CreateFrame(kDontCare, kDontCare, kDontCare));
5613
5614 using Config = EncoderSwitchRequestCallback::Config;
5615 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5616 .Times(0);
5617
Henrik Boström381d1092020-05-12 18:49:07 +02005618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005619 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5620 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5621 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5622 /*fraction_lost=*/0,
5623 /*rtt_ms=*/0,
5624 /*cwnd_reduce_ratio=*/0);
5625
5626 video_stream_encoder_->Stop();
5627}
5628
philipeld9cc8c02019-09-16 14:53:40 +02005629TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5630 constexpr int kSufficientBitrateToNotDrop = 1000;
5631 constexpr int kHighRes = 500;
5632 constexpr int kLowRes = 100;
5633
5634 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5635 video_send_config_.encoder_settings.encoder_switch_request_callback =
5636 &switch_callback;
5637 webrtc::test::ScopedFieldTrials field_trial(
5638 "WebRTC-NetworkCondition-EncoderSwitch/"
5639 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5640 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5641 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5642 encoder_config.codec_type = kVideoCodecH264;
5643
5644 // Reset encoder for new configuration to take effect.
5645 ConfigureEncoder(std::move(encoder_config));
5646
5647 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5648 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5649 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005650 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005651 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5652 /*stable_target_bitrate=*/
5653 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5654 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005655 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005656 /*rtt_ms=*/0,
5657 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005658
5659 // Send one frame to trigger ReconfigureEncoder.
5660 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5661 WaitForEncodedFrame(1);
5662
5663 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005664 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5665 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005666 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005667 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005668
5669 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5670 WaitForEncodedFrame(2);
5671
5672 video_stream_encoder_->Stop();
5673}
5674
philipel9b058032020-02-10 11:30:00 +01005675TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5676 constexpr int kDontCare = 100;
5677 StrictMock<MockEncoderSelector> encoder_selector;
5678 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5679 &fake_encoder_, &encoder_selector);
5680 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5681
5682 // Reset encoder for new configuration to take effect.
5683 ConfigureEncoder(video_encoder_config_.Copy());
5684
5685 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5686
5687 video_source_.IncomingCapturedFrame(
5688 CreateFrame(kDontCare, kDontCare, kDontCare));
5689 video_stream_encoder_->Stop();
5690
5691 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5692 // to it's factory, so in order for the encoder instance in the
5693 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5694 // reset the |video_stream_encoder_| here.
5695 video_stream_encoder_.reset();
5696}
5697
5698TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5699 constexpr int kDontCare = 100;
5700
5701 NiceMock<MockEncoderSelector> encoder_selector;
5702 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5703 video_send_config_.encoder_settings.encoder_switch_request_callback =
5704 &switch_callback;
5705 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5706 &fake_encoder_, &encoder_selector);
5707 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5708
5709 // Reset encoder for new configuration to take effect.
5710 ConfigureEncoder(video_encoder_config_.Copy());
5711
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005712 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005713 .WillByDefault(Return(SdpVideoFormat("AV1")));
5714 EXPECT_CALL(switch_callback,
5715 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5716 Field(&SdpVideoFormat::name, "AV1"))));
5717
Henrik Boström381d1092020-05-12 18:49:07 +02005718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005719 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5720 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5721 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005722 /*fraction_lost=*/0,
5723 /*rtt_ms=*/0,
5724 /*cwnd_reduce_ratio=*/0);
5725
5726 video_stream_encoder_->Stop();
5727}
5728
5729TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5730 constexpr int kSufficientBitrateToNotDrop = 1000;
5731 constexpr int kDontCare = 100;
5732
5733 NiceMock<MockVideoEncoder> video_encoder;
5734 NiceMock<MockEncoderSelector> encoder_selector;
5735 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5736 video_send_config_.encoder_settings.encoder_switch_request_callback =
5737 &switch_callback;
5738 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5739 &video_encoder, &encoder_selector);
5740 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5741
5742 // Reset encoder for new configuration to take effect.
5743 ConfigureEncoder(video_encoder_config_.Copy());
5744
5745 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5746 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5747 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005749 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5750 /*stable_target_bitrate=*/
5751 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5752 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005753 /*fraction_lost=*/0,
5754 /*rtt_ms=*/0,
5755 /*cwnd_reduce_ratio=*/0);
5756
5757 ON_CALL(video_encoder, Encode(_, _))
5758 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5759 ON_CALL(encoder_selector, OnEncoderBroken())
5760 .WillByDefault(Return(SdpVideoFormat("AV2")));
5761
5762 rtc::Event encode_attempted;
5763 EXPECT_CALL(switch_callback,
5764 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5765 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5766 EXPECT_EQ(format.name, "AV2");
5767 encode_attempted.Set();
5768 });
5769
5770 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5771 encode_attempted.Wait(3000);
5772
5773 video_stream_encoder_->Stop();
5774
5775 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5776 // to it's factory, so in order for the encoder instance in the
5777 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5778 // reset the |video_stream_encoder_| here.
5779 video_stream_encoder_.reset();
5780}
5781
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005782TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005783 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005784 const int kFrameWidth = 320;
5785 const int kFrameHeight = 180;
5786
5787 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005788 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005790 /*target_bitrate=*/rate,
5791 /*stable_target_bitrate=*/rate,
5792 /*link_allocation=*/rate,
5793 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005794 /*rtt_ms=*/0,
5795 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005796
5797 // Insert a first video frame so that encoder gets configured.
5798 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5799 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5800 frame.set_rotation(kVideoRotation_270);
5801 video_source_.IncomingCapturedFrame(frame);
5802 WaitForEncodedFrame(timestamp_ms);
5803 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5804
5805 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005806 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005808 /*target_bitrate=*/new_stable_rate,
5809 /*stable_target_bitrate=*/new_stable_rate,
5810 /*link_allocation=*/rate,
5811 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005812 /*rtt_ms=*/0,
5813 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005814 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5815 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5816 video_stream_encoder_->Stop();
5817}
5818
5819TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005820 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005821 const int kFrameWidth = 320;
5822 const int kFrameHeight = 180;
5823
5824 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005825 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005827 /*target_bitrate=*/rate,
5828 /*stable_target_bitrate=*/rate,
5829 /*link_allocation=*/rate,
5830 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005831 /*rtt_ms=*/0,
5832 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005833
5834 // Insert a first video frame so that encoder gets configured.
5835 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5836 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5837 frame.set_rotation(kVideoRotation_270);
5838 video_source_.IncomingCapturedFrame(frame);
5839 WaitForEncodedFrame(timestamp_ms);
5840 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5841
5842 // Set a higher target rate without changing the link_allocation. Should not
5843 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005844 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005845 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005846 /*target_bitrate=*/rate,
5847 /*stable_target_bitrate=*/new_stable_rate,
5848 /*link_allocation=*/rate,
5849 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005850 /*rtt_ms=*/0,
5851 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005852 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5853 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5854 video_stream_encoder_->Stop();
5855}
5856
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005857TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5858 test::ScopedFieldTrials field_trials(
5859 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5860 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5861 const int kFramerateFps = 30;
5862 const int kWidth = 1920;
5863 const int kHeight = 1080;
5864 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5865 // Works on screenshare mode.
5866 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5867 // We rely on the automatic resolution adaptation, but we handle framerate
5868 // adaptation manually by mocking the stats proxy.
5869 video_source_.set_adaptation_enabled(true);
5870
5871 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005873 DataRate::BitsPerSec(kTargetBitrateBps),
5874 DataRate::BitsPerSec(kTargetBitrateBps),
5875 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005876 video_stream_encoder_->SetSource(&video_source_,
5877 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005878 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005879
5880 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5881 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5882
5883 // Pass enough frames with the full update to trigger animation detection.
5884 for (int i = 0; i < kNumFrames; ++i) {
5885 int64_t timestamp_ms =
5886 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5887 frame.set_ntp_time_ms(timestamp_ms);
5888 frame.set_timestamp_us(timestamp_ms * 1000);
5889 video_source_.IncomingCapturedFrame(frame);
5890 WaitForEncodedFrame(timestamp_ms);
5891 }
5892
5893 // Resolution should be limited.
5894 rtc::VideoSinkWants expected;
5895 expected.max_framerate_fps = kFramerateFps;
5896 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005897 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005898
5899 // Pass one frame with no known update.
5900 // Resolution cap should be removed immediately.
5901 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5902 frame.set_ntp_time_ms(timestamp_ms);
5903 frame.set_timestamp_us(timestamp_ms * 1000);
5904 frame.clear_update_rect();
5905
5906 video_source_.IncomingCapturedFrame(frame);
5907 WaitForEncodedFrame(timestamp_ms);
5908
5909 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005910 EXPECT_THAT(video_source_.sink_wants(),
5911 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005912
5913 video_stream_encoder_->Stop();
5914}
5915
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02005916TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
5917 const int kWidth = 720; // 540p adapted down.
5918 const int kHeight = 405;
5919 const int kNumFrames = 3;
5920 // Works on screenshare mode.
5921 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5922 /*num_spatial_layers=*/2, /*screenshare=*/true);
5923
5924 video_source_.set_adaptation_enabled(true);
5925
5926 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5927 DataRate::BitsPerSec(kTargetBitrateBps),
5928 DataRate::BitsPerSec(kTargetBitrateBps),
5929 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5930
5931 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5932
5933 // Pass enough frames with the full update to trigger animation detection.
5934 for (int i = 0; i < kNumFrames; ++i) {
5935 int64_t timestamp_ms =
5936 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5937 frame.set_ntp_time_ms(timestamp_ms);
5938 frame.set_timestamp_us(timestamp_ms * 1000);
5939 video_source_.IncomingCapturedFrame(frame);
5940 WaitForEncodedFrame(timestamp_ms);
5941 }
5942
5943 video_stream_encoder_->Stop();
5944}
5945
perkj26091b12016-09-01 01:17:40 -07005946} // namespace webrtc