blob: c0ff85ea85daffff26f6931ee364084da91737ef [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020029#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010030#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020031#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070032#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020034#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020035#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020036#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010037#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020039#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020042#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010043#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020044#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020045#include "system_wrappers/include/sleep.h"
46#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020052#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020053#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070054
55namespace webrtc {
56
sprang57c2fff2017-01-16 06:24:02 -080057using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020058using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020059using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020060using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Ge;
62using ::testing::Gt;
63using ::testing::Le;
64using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010065using ::testing::Matcher;
66using ::testing::NiceMock;
67using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020068using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080069
perkj803d97f2016-11-01 11:45:46 -070070namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020071const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010072const int kQpLow = 1;
73const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinFramerateFps = 2;
75const int kMinBalancedFramerateFps = 7;
76const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080077const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010078const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020079const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010080const uint32_t kSimulcastTargetBitrateBps = 3150000;
81const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080082const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070083const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020084const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020085const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020086const VideoEncoder::ResolutionBitrateLimits
87 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
88const VideoEncoder::ResolutionBitrateLimits
89 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080090
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020091uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
92 0x00, 0x00, 0x03, 0x03, 0xF4,
93 0x05, 0x03, 0xC7, 0xE0, 0x1B,
94 0x41, 0x10, 0x8D, 0x00};
95
perkj803d97f2016-11-01 11:45:46 -070096class TestBuffer : public webrtc::I420Buffer {
97 public:
98 TestBuffer(rtc::Event* event, int width, int height)
99 : I420Buffer(width, height), event_(event) {}
100
101 private:
102 friend class rtc::RefCountedObject<TestBuffer>;
103 ~TestBuffer() override {
104 if (event_)
105 event_->Set();
106 }
107 rtc::Event* const event_;
108};
109
Noah Richards51db4212019-06-12 06:59:12 -0700110// A fake native buffer that can't be converted to I420.
111class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
112 public:
113 FakeNativeBuffer(rtc::Event* event, int width, int height)
114 : event_(event), width_(width), height_(height) {}
115 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
116 int width() const override { return width_; }
117 int height() const override { return height_; }
118 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
119 return nullptr;
120 }
121
122 private:
123 friend class rtc::RefCountedObject<FakeNativeBuffer>;
124 ~FakeNativeBuffer() override {
125 if (event_)
126 event_->Set();
127 }
128 rtc::Event* const event_;
129 const int width_;
130 const int height_;
131};
132
Niels Möller7dc26b72017-12-06 10:27:48 +0100133class CpuOveruseDetectorProxy : public OveruseFrameDetector {
134 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200135 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
136 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200137 last_target_framerate_fps_(-1),
138 framerate_updated_event_(true /* manual_reset */,
139 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100140 virtual ~CpuOveruseDetectorProxy() {}
141
142 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200143 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 last_target_framerate_fps_ = framerate_fps;
145 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200146 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100147 }
148
149 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200150 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100151 return last_target_framerate_fps_;
152 }
153
Niels Möller4db138e2018-04-19 09:04:13 +0200154 CpuOveruseOptions GetOptions() { return options_; }
155
Henrik Boström381d1092020-05-12 18:49:07 +0200156 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
157
Niels Möller7dc26b72017-12-06 10:27:48 +0100158 private:
Markus Handella3765182020-07-08 13:13:32 +0200159 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100160 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200161 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100162};
163
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200164class FakeVideoSourceRestrictionsListener
165 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200166 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200167 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200168 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200169 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200170 RTC_DCHECK(was_restrictions_updated_);
171 }
172
173 rtc::Event* restrictions_updated_event() {
174 return &restrictions_updated_event_;
175 }
176
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200177 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200178 void OnVideoSourceRestrictionsUpdated(
179 VideoSourceRestrictions restrictions,
180 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200181 rtc::scoped_refptr<Resource> reason,
182 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200183 was_restrictions_updated_ = true;
184 restrictions_updated_event_.Set();
185 }
186
187 private:
188 bool was_restrictions_updated_;
189 rtc::Event restrictions_updated_event_;
190};
191
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200192auto WantsFps(Matcher<int> fps_matcher) {
193 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
194 fps_matcher);
195}
196
197auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
198 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
199 AllOf(max_pixel_matcher, Gt(0)));
200}
201
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200202auto ResolutionMax() {
203 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200204 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200205 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
206 Eq(absl::nullopt)));
207}
208
209auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200210 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200211}
212
213auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200214 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200215}
216
217auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200218 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200219}
220
221auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200222 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200223}
224
225auto FpsMaxResolutionMax() {
226 return AllOf(FpsMax(), ResolutionMax());
227}
228
229auto UnlimitedSinkWants() {
230 return AllOf(FpsUnlimited(), ResolutionMax());
231}
232
233auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
234 Matcher<int> fps_range_matcher;
235
236 if (last_frame_pixels <= 320 * 240) {
237 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200238 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 fps_range_matcher = AllOf(Ge(10), Le(15));
240 } else if (last_frame_pixels <= 640 * 480) {
241 fps_range_matcher = Ge(15);
242 } else {
243 fps_range_matcher = Eq(kDefaultFramerate);
244 }
245 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
246 fps_range_matcher);
247}
248
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200249auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
250 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
251 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
252}
253
254auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
255 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
256}
257
258auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
259 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
260}
261
262auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
263 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
264 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
265}
266
267auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
268 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
269 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
270}
271
272auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
273 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
274 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
275}
276
277auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
278 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
279 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
280}
281
mflodmancc3d4422017-08-03 08:27:51 -0700282class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700283 public:
Niels Möller213618e2018-07-24 09:29:58 +0200284 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200285 const VideoStreamEncoderSettings& settings,
286 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100287 : VideoStreamEncoder(Clock::GetRealTimeClock(),
288 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200289 stats_proxy,
290 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200291 std::unique_ptr<OveruseFrameDetector>(
292 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100293 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100294 task_queue_factory),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200295 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200296 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200297 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200298 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200299 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200300 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200301 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100302 }
perkj803d97f2016-11-01 11:45:46 -0700303
Henrik Boström381d1092020-05-12 18:49:07 +0200304 void SetSourceAndWaitForRestrictionsUpdated(
305 rtc::VideoSourceInterface<VideoFrame>* source,
306 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200307 FakeVideoSourceRestrictionsListener listener;
308 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200309 SetSource(source, degradation_preference);
310 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200311 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200312 }
313
314 void SetSourceAndWaitForFramerateUpdated(
315 rtc::VideoSourceInterface<VideoFrame>* source,
316 const DegradationPreference& degradation_preference) {
317 overuse_detector_proxy_->framerate_updated_event()->Reset();
318 SetSource(source, degradation_preference);
319 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
320 }
321
322 void OnBitrateUpdatedAndWaitForManagedResources(
323 DataRate target_bitrate,
324 DataRate stable_target_bitrate,
325 DataRate link_allocation,
326 uint8_t fraction_lost,
327 int64_t round_trip_time_ms,
328 double cwnd_reduce_ratio) {
329 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
330 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
331 // Bitrate is updated on the encoder queue.
332 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200333 }
334
kthelgason2fc52542017-03-03 00:24:41 -0800335 // This is used as a synchronisation mechanism, to make sure that the
336 // encoder queue is not blocked before we start sending it frames.
337 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100338 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200339 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800340 ASSERT_TRUE(event.Wait(5000));
341 }
342
Henrik Boström91aa7322020-04-28 12:24:33 +0200343 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200344 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200345 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200346 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200347 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200348 event.Set();
349 });
350 ASSERT_TRUE(event.Wait(5000));
351 }
352 void TriggerCpuUnderuse() {
353 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200354 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200355 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200356 event.Set();
357 });
358 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200359 }
kthelgason876222f2016-11-29 01:44:11 -0800360
Henrik Boström91aa7322020-04-28 12:24:33 +0200361 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200362 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200363 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200364 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200365 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200366 event.Set();
367 });
368 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200369 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200370 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200371 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200372 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200374 event.Set();
375 });
376 ASSERT_TRUE(event.Wait(5000));
377 }
378
Niels Möller7dc26b72017-12-06 10:27:48 +0100379 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200380 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
381 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200382 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700383};
384
asapersson5f7226f2016-11-25 04:37:00 -0800385class VideoStreamFactory
386 : public VideoEncoderConfig::VideoStreamFactoryInterface {
387 public:
sprangfda496a2017-06-15 04:21:07 -0700388 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
389 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800390 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700391 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800392 }
393
394 private:
395 std::vector<VideoStream> CreateEncoderStreams(
396 int width,
397 int height,
398 const VideoEncoderConfig& encoder_config) override {
399 std::vector<VideoStream> streams =
400 test::CreateVideoStreams(width, height, encoder_config);
401 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100402 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700403 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800404 }
405 return streams;
406 }
sprangfda496a2017-06-15 04:21:07 -0700407
asapersson5f7226f2016-11-25 04:37:00 -0800408 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700409 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800410};
411
Noah Richards51db4212019-06-12 06:59:12 -0700412// Simulates simulcast behavior and makes highest stream resolutions divisible
413// by 4.
414class CroppingVideoStreamFactory
415 : public VideoEncoderConfig::VideoStreamFactoryInterface {
416 public:
417 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
418 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
419 EXPECT_GT(num_temporal_layers, 0u);
420 EXPECT_GT(framerate, 0);
421 }
422
423 private:
424 std::vector<VideoStream> CreateEncoderStreams(
425 int width,
426 int height,
427 const VideoEncoderConfig& encoder_config) override {
428 std::vector<VideoStream> streams = test::CreateVideoStreams(
429 width - width % 4, height - height % 4, encoder_config);
430 for (VideoStream& stream : streams) {
431 stream.num_temporal_layers = num_temporal_layers_;
432 stream.max_framerate = framerate_;
433 }
434 return streams;
435 }
436
437 const size_t num_temporal_layers_;
438 const int framerate_;
439};
440
sprangb1ca0732017-02-01 08:38:12 -0800441class AdaptingFrameForwarder : public test::FrameForwarder {
442 public:
443 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700444 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800445
446 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200447 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800448 adaptation_enabled_ = enabled;
449 }
450
asaperssonfab67072017-04-04 05:51:49 -0700451 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200452 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800453 return adaptation_enabled_;
454 }
455
asapersson09f05612017-05-15 23:40:18 -0700456 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200457 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700458 return last_wants_;
459 }
460
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200461 absl::optional<int> last_sent_width() const { return last_width_; }
462 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800463
sprangb1ca0732017-02-01 08:38:12 -0800464 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
465 int cropped_width = 0;
466 int cropped_height = 0;
467 int out_width = 0;
468 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700469 if (adaption_enabled()) {
470 if (adapter_.AdaptFrameResolution(
471 video_frame.width(), video_frame.height(),
472 video_frame.timestamp_us() * 1000, &cropped_width,
473 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100474 VideoFrame adapted_frame =
475 VideoFrame::Builder()
476 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
477 nullptr, out_width, out_height))
478 .set_timestamp_rtp(99)
479 .set_timestamp_ms(99)
480 .set_rotation(kVideoRotation_0)
481 .build();
sprangc5d62e22017-04-02 23:53:04 -0700482 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100483 if (video_frame.has_update_rect()) {
484 adapted_frame.set_update_rect(
485 video_frame.update_rect().ScaleWithFrame(
486 video_frame.width(), video_frame.height(), 0, 0,
487 video_frame.width(), video_frame.height(), out_width,
488 out_height));
489 }
sprangc5d62e22017-04-02 23:53:04 -0700490 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800491 last_width_.emplace(adapted_frame.width());
492 last_height_.emplace(adapted_frame.height());
493 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200494 last_width_ = absl::nullopt;
495 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700496 }
sprangb1ca0732017-02-01 08:38:12 -0800497 } else {
498 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800499 last_width_.emplace(video_frame.width());
500 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800501 }
502 }
503
504 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
505 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200506 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200507 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100508 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200509 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800510 }
sprangb1ca0732017-02-01 08:38:12 -0800511 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200512 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
513 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200514 absl::optional<int> last_width_;
515 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800516};
sprangc5d62e22017-04-02 23:53:04 -0700517
Niels Möller213618e2018-07-24 09:29:58 +0200518// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700519class MockableSendStatisticsProxy : public SendStatisticsProxy {
520 public:
521 MockableSendStatisticsProxy(Clock* clock,
522 const VideoSendStream::Config& config,
523 VideoEncoderConfig::ContentType content_type)
524 : SendStatisticsProxy(clock, config, content_type) {}
525
526 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200527 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700528 if (mock_stats_)
529 return *mock_stats_;
530 return SendStatisticsProxy::GetStats();
531 }
532
Niels Möller213618e2018-07-24 09:29:58 +0200533 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200534 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200535 if (mock_stats_)
536 return mock_stats_->input_frame_rate;
537 return SendStatisticsProxy::GetInputFrameRate();
538 }
sprangc5d62e22017-04-02 23:53:04 -0700539 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200540 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700541 mock_stats_.emplace(stats);
542 }
543
544 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200545 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700546 mock_stats_.reset();
547 }
548
549 private:
Markus Handella3765182020-07-08 13:13:32 +0200550 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200551 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700552};
553
sprang4847ae62017-06-27 07:06:52 -0700554class MockBitrateObserver : public VideoBitrateAllocationObserver {
555 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200556 MOCK_METHOD(void,
557 OnBitrateAllocationUpdated,
558 (const VideoBitrateAllocation&),
559 (override));
sprang4847ae62017-06-27 07:06:52 -0700560};
561
philipel9b058032020-02-10 11:30:00 +0100562class MockEncoderSelector
563 : public VideoEncoderFactory::EncoderSelectorInterface {
564 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200565 MOCK_METHOD(void,
566 OnCurrentEncoder,
567 (const SdpVideoFormat& format),
568 (override));
569 MOCK_METHOD(absl::optional<SdpVideoFormat>,
570 OnAvailableBitrate,
571 (const DataRate& rate),
572 (override));
573 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100574};
575
perkj803d97f2016-11-01 11:45:46 -0700576} // namespace
577
mflodmancc3d4422017-08-03 08:27:51 -0700578class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700579 public:
580 static const int kDefaultTimeoutMs = 30 * 1000;
581
mflodmancc3d4422017-08-03 08:27:51 -0700582 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700583 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700584 codec_width_(320),
585 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200586 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200587 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700588 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200589 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700590 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700591 Clock::GetRealTimeClock(),
592 video_send_config_,
593 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700594 sink_(&fake_encoder_) {}
595
596 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700597 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700598 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200599 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800600 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200601 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200602 video_send_config_.rtp.payload_name = "FAKE";
603 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700604
Per512ecb32016-09-23 15:52:06 +0200605 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200606 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700607 video_encoder_config.video_stream_factory =
608 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100609 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700610
611 // Framerate limit is specified by the VideoStreamFactory.
612 std::vector<VideoStream> streams =
613 video_encoder_config.video_stream_factory->CreateEncoderStreams(
614 codec_width_, codec_height_, video_encoder_config);
615 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100616 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700617
Niels Möllerf1338562018-04-26 09:51:47 +0200618 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800619 }
620
Niels Möllerf1338562018-04-26 09:51:47 +0200621 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700622 if (video_stream_encoder_)
623 video_stream_encoder_->Stop();
624 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200625 stats_proxy_.get(), video_send_config_.encoder_settings,
626 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700627 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
628 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700629 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700630 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
631 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200632 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700633 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800634 }
635
636 void ResetEncoder(const std::string& payload_name,
637 size_t num_streams,
638 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700639 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700640 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200641 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800642
643 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200644 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800645 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100646 video_encoder_config.max_bitrate_bps =
647 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800648 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700649 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
650 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700651 video_encoder_config.content_type =
652 screenshare ? VideoEncoderConfig::ContentType::kScreen
653 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700654 if (payload_name == "VP9") {
655 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
656 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200657 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700658 video_encoder_config.encoder_specific_settings =
659 new rtc::RefCountedObject<
660 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
661 }
Niels Möllerf1338562018-04-26 09:51:47 +0200662 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700663 }
664
sprang57c2fff2017-01-16 06:24:02 -0800665 VideoFrame CreateFrame(int64_t ntp_time_ms,
666 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100667 VideoFrame frame =
668 VideoFrame::Builder()
669 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
670 destruction_event, codec_width_, codec_height_))
671 .set_timestamp_rtp(99)
672 .set_timestamp_ms(99)
673 .set_rotation(kVideoRotation_0)
674 .build();
sprang57c2fff2017-01-16 06:24:02 -0800675 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700676 return frame;
677 }
678
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100679 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
680 rtc::Event* destruction_event,
681 int offset_x) const {
682 VideoFrame frame =
683 VideoFrame::Builder()
684 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
685 destruction_event, codec_width_, codec_height_))
686 .set_timestamp_rtp(99)
687 .set_timestamp_ms(99)
688 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100689 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100690 .build();
691 frame.set_ntp_time_ms(ntp_time_ms);
692 return frame;
693 }
694
sprang57c2fff2017-01-16 06:24:02 -0800695 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100696 VideoFrame frame =
697 VideoFrame::Builder()
698 .set_video_frame_buffer(
699 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
700 .set_timestamp_rtp(99)
701 .set_timestamp_ms(99)
702 .set_rotation(kVideoRotation_0)
703 .build();
sprang57c2fff2017-01-16 06:24:02 -0800704 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700705 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700706 return frame;
707 }
708
Noah Richards51db4212019-06-12 06:59:12 -0700709 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
710 rtc::Event* destruction_event,
711 int width,
712 int height) const {
713 VideoFrame frame =
714 VideoFrame::Builder()
715 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
716 destruction_event, width, height))
717 .set_timestamp_rtp(99)
718 .set_timestamp_ms(99)
719 .set_rotation(kVideoRotation_0)
720 .build();
721 frame.set_ntp_time_ms(ntp_time_ms);
722 return frame;
723 }
724
725 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
726 rtc::Event* destruction_event) const {
727 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
728 codec_height_);
729 }
730
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100731 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
732 MockBitrateObserver bitrate_observer;
733 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
734
735 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
736 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100738 DataRate::BitsPerSec(kTargetBitrateBps),
739 DataRate::BitsPerSec(kTargetBitrateBps),
740 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100741
742 video_source_.IncomingCapturedFrame(
743 CreateFrame(1, codec_width_, codec_height_));
744 WaitForEncodedFrame(1);
745 }
746
sprang4847ae62017-06-27 07:06:52 -0700747 void WaitForEncodedFrame(int64_t expected_ntp_time) {
748 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100749 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700750 }
751
752 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
753 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100754 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700755 return ok;
756 }
757
758 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
759 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100760 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700761 }
762
763 void ExpectDroppedFrame() {
764 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100765 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700766 }
767
768 bool WaitForFrame(int64_t timeout_ms) {
769 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100770 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700771 return ok;
772 }
773
perkj26091b12016-09-01 01:17:40 -0700774 class TestEncoder : public test::FakeEncoder {
775 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100776 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700777
asaperssonfab67072017-04-04 05:51:49 -0700778 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200779 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700780 return config_;
781 }
782
783 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200784 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700785 block_next_encode_ = true;
786 }
787
Erik Språngaed30702018-11-05 12:57:17 +0100788 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200789 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100790 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100791 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100792 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100793 info.scaling_settings = VideoEncoder::ScalingSettings(
794 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100795 }
796 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100797 for (int i = 0; i < kMaxSpatialLayers; ++i) {
798 if (temporal_layers_supported_[i]) {
799 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
800 info.fps_allocation[i].resize(num_layers);
801 }
802 }
Erik Språngaed30702018-11-05 12:57:17 +0100803 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200804
805 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100806 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200807 info.apply_alignment_to_all_simulcast_layers =
808 apply_alignment_to_all_simulcast_layers_;
Erik Språngaed30702018-11-05 12:57:17 +0100809 return info;
kthelgason876222f2016-11-29 01:44:11 -0800810 }
811
Erik Språngb7cb7b52019-02-26 15:52:33 +0100812 int32_t RegisterEncodeCompleteCallback(
813 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200814 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100815 encoded_image_callback_ = callback;
816 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
817 }
818
perkjfa10b552016-10-02 23:45:26 -0700819 void ContinueEncode() { continue_encode_event_.Set(); }
820
821 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
822 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200823 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700824 EXPECT_EQ(timestamp_, timestamp);
825 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
826 }
827
kthelgason2fc52542017-03-03 00:24:41 -0800828 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200829 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800830 quality_scaling_ = b;
831 }
kthelgasonad9010c2017-02-14 00:46:51 -0800832
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100833 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200834 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100835 requested_resolution_alignment_ = requested_resolution_alignment;
836 }
837
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200838 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
839 MutexLock lock(&local_mutex_);
840 apply_alignment_to_all_simulcast_layers_ = b;
841 }
842
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100843 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200844 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100845 is_hardware_accelerated_ = is_hardware_accelerated;
846 }
847
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100848 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
849 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200850 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100851 temporal_layers_supported_[spatial_idx] = supported;
852 }
853
Sergey Silkin6456e352019-07-08 17:56:40 +0200854 void SetResolutionBitrateLimits(
855 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200856 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200857 resolution_bitrate_limits_ = thresholds;
858 }
859
sprangfe627f32017-03-29 08:24:59 -0700860 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200861 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700862 force_init_encode_failed_ = force_failure;
863 }
864
Niels Möller6bb5ab92019-01-11 11:11:10 +0100865 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200866 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100867 rate_factor_ = rate_factor;
868 }
869
Erik Språngd7329ca2019-02-21 21:19:53 +0100870 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200871 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100872 return last_framerate_;
873 }
874
Erik Språngd7329ca2019-02-21 21:19:53 +0100875 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200876 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100877 return last_update_rect_;
878 }
879
Niels Möller87e2d782019-03-07 10:18:23 +0100880 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200881 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100882 return last_frame_types_;
883 }
884
885 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100886 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100887 keyframe ? VideoFrameType::kVideoFrameKey
888 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100889 {
Markus Handella3765182020-07-08 13:13:32 +0200890 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100891 last_frame_types_ = frame_type;
892 }
Niels Möllerb859b322019-03-07 12:40:01 +0100893 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100894 }
895
Erik Språngb7cb7b52019-02-26 15:52:33 +0100896 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200897 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200898 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100899 }
900
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200901 void SetEncodedImageData(
902 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200903 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200904 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200905 }
906
Erik Språngd7329ca2019-02-21 21:19:53 +0100907 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200908 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100909 expect_null_frame_ = true;
910 }
911
Erik Språng5056af02019-09-02 15:53:11 +0200912 absl::optional<VideoEncoder::RateControlParameters>
913 GetAndResetLastRateControlSettings() {
914 auto settings = last_rate_control_settings_;
915 last_rate_control_settings_.reset();
916 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100917 }
918
Sergey Silkin5ee69672019-07-02 14:18:34 +0200919 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200920 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200921 return num_encoder_initializations_;
922 }
923
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200924 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200925 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200926 return num_set_rates_;
927 }
928
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200929 VideoCodec video_codec() const {
930 MutexLock lock(&local_mutex_);
931 return video_codec_;
932 }
933
perkjfa10b552016-10-02 23:45:26 -0700934 private:
perkj26091b12016-09-01 01:17:40 -0700935 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100936 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700937 bool block_encode;
938 {
Markus Handella3765182020-07-08 13:13:32 +0200939 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100940 if (expect_null_frame_) {
941 EXPECT_EQ(input_image.timestamp(), 0u);
942 EXPECT_EQ(input_image.width(), 1);
943 last_frame_types_ = *frame_types;
944 expect_null_frame_ = false;
945 } else {
946 EXPECT_GT(input_image.timestamp(), timestamp_);
947 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
948 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
949 }
perkj26091b12016-09-01 01:17:40 -0700950
951 timestamp_ = input_image.timestamp();
952 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700953 last_input_width_ = input_image.width();
954 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700955 block_encode = block_next_encode_;
956 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100957 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100958 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700959 }
Niels Möllerb859b322019-03-07 12:40:01 +0100960 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700961 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700962 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700963 return result;
964 }
965
Danil Chapovalov2549f172020-08-12 17:30:36 +0200966 CodecSpecificInfo EncodeHook(EncodedImage& encoded_image) override {
967 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200968 {
969 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200970 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200971 }
972 MutexLock lock(&local_mutex_);
973 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +0200974 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200975 }
Danil Chapovalov2549f172020-08-12 17:30:36 +0200976 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200977 }
978
sprangfe627f32017-03-29 08:24:59 -0700979 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200980 const Settings& settings) override {
981 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200982
Markus Handella3765182020-07-08 13:13:32 +0200983 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100984 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200985
986 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200987 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +0200988
Erik Språng82fad3d2018-03-21 09:57:23 +0100989 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700990 // Simulate setting up temporal layers, in order to validate the life
991 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100992 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200993 frame_buffer_controller_ =
994 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700995 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100996 if (force_init_encode_failed_) {
997 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -0700998 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100999 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001000
Erik Språngb7cb7b52019-02-26 15:52:33 +01001001 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001002 return res;
1003 }
1004
Erik Språngb7cb7b52019-02-26 15:52:33 +01001005 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001006 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001007 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1008 initialized_ = EncoderState::kUninitialized;
1009 return FakeEncoder::Release();
1010 }
1011
Erik Språng16cb8f52019-04-12 13:59:09 +02001012 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001013 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001014 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001015 VideoBitrateAllocation adjusted_rate_allocation;
1016 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1017 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001018 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001019 adjusted_rate_allocation.SetBitrate(
1020 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001021 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001022 rate_factor_));
1023 }
1024 }
1025 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001026 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001027 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001028 RateControlParameters adjusted_paramters = parameters;
1029 adjusted_paramters.bitrate = adjusted_rate_allocation;
1030 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001031 }
1032
Markus Handella3765182020-07-08 13:13:32 +02001033 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001034 enum class EncoderState {
1035 kUninitialized,
1036 kInitializationFailed,
1037 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001038 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1039 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001040 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001041 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1042 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1043 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1044 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1045 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1046 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001047 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1048 false;
Markus Handella3765182020-07-08 13:13:32 +02001049 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001050 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1051 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001052 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001053 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001054 absl::optional<bool>
1055 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001056 local_mutex_);
1057 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1058 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1059 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001060 absl::optional<VideoEncoder::RateControlParameters>
1061 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001062 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1063 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001064 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001065 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001066 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1067 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001068 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001069 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001070 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001071 RTC_GUARDED_BY(local_mutex_);
1072 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001073 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001074 };
1075
mflodmancc3d4422017-08-03 08:27:51 -07001076 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001077 public:
1078 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001079 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001080
perkj26091b12016-09-01 01:17:40 -07001081 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001082 EXPECT_TRUE(
1083 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1084 }
1085
1086 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1087 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001088 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001089 if (!encoded_frame_event_.Wait(timeout_ms))
1090 return false;
perkj26091b12016-09-01 01:17:40 -07001091 {
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001093 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001094 }
1095 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001096 return true;
perkj26091b12016-09-01 01:17:40 -07001097 }
1098
sprangb1ca0732017-02-01 08:38:12 -08001099 void WaitForEncodedFrame(uint32_t expected_width,
1100 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001101 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001102 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001103 }
1104
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001105 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001106 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001107 uint32_t width = 0;
1108 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001109 {
Markus Handella3765182020-07-08 13:13:32 +02001110 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001111 width = last_width_;
1112 height = last_height_;
1113 }
1114 EXPECT_EQ(expected_height, height);
1115 EXPECT_EQ(expected_width, width);
1116 }
1117
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001118 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1119 VideoRotation rotation;
1120 {
Markus Handella3765182020-07-08 13:13:32 +02001121 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001122 rotation = last_rotation_;
1123 }
1124 EXPECT_EQ(expected_rotation, rotation);
1125 }
1126
kthelgason2fc52542017-03-03 00:24:41 -08001127 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001128
sprangc5d62e22017-04-02 23:53:04 -07001129 bool WaitForFrame(int64_t timeout_ms) {
1130 return encoded_frame_event_.Wait(timeout_ms);
1131 }
1132
perkj26091b12016-09-01 01:17:40 -07001133 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001134 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001135 expect_frames_ = false;
1136 }
1137
asaperssonfab67072017-04-04 05:51:49 -07001138 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001139 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001140 return number_of_reconfigurations_;
1141 }
1142
asaperssonfab67072017-04-04 05:51:49 -07001143 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001144 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001145 return min_transmit_bitrate_bps_;
1146 }
1147
Erik Språngd7329ca2019-02-21 21:19:53 +01001148 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001149 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001150 num_expected_layers_ = num_layers;
1151 }
1152
Erik Språngb7cb7b52019-02-26 15:52:33 +01001153 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001154 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001155 return last_capture_time_ms_;
1156 }
1157
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001158 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001159 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001160 return std::move(last_encoded_image_data_);
1161 }
1162
perkj26091b12016-09-01 01:17:40 -07001163 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001164 Result OnEncodedImage(
1165 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001166 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001167 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001168 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001169 last_encoded_image_data_ = std::vector<uint8_t>(
1170 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001171 uint32_t timestamp = encoded_image.Timestamp();
1172 if (last_timestamp_ != timestamp) {
1173 num_received_layers_ = 1;
1174 } else {
1175 ++num_received_layers_;
1176 }
1177 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001178 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001179 last_width_ = encoded_image._encodedWidth;
1180 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001181 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001182 if (num_received_layers_ == num_expected_layers_) {
1183 encoded_frame_event_.Set();
1184 }
sprangb1ca0732017-02-01 08:38:12 -08001185 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001186 }
1187
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001188 void OnEncoderConfigurationChanged(
1189 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001190 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001191 VideoEncoderConfig::ContentType content_type,
1192 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001193 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001194 ++number_of_reconfigurations_;
1195 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1196 }
1197
Markus Handella3765182020-07-08 13:13:32 +02001198 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001199 TestEncoder* test_encoder_;
1200 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001201 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001202 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001203 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001204 uint32_t last_height_ = 0;
1205 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001206 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001207 size_t num_expected_layers_ = 1;
1208 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001209 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001210 int number_of_reconfigurations_ = 0;
1211 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001212 };
1213
Sergey Silkin5ee69672019-07-02 14:18:34 +02001214 class VideoBitrateAllocatorProxyFactory
1215 : public VideoBitrateAllocatorFactory {
1216 public:
1217 VideoBitrateAllocatorProxyFactory()
1218 : bitrate_allocator_factory_(
1219 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1220
1221 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1222 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001223 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001224 codec_config_ = codec;
1225 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1226 }
1227
1228 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001229 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001230 return codec_config_;
1231 }
1232
1233 private:
1234 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1235
Markus Handella3765182020-07-08 13:13:32 +02001236 mutable Mutex mutex_;
1237 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001238 };
1239
perkj26091b12016-09-01 01:17:40 -07001240 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001241 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001242 int codec_width_;
1243 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001244 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001245 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001246 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001247 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001248 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001249 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001250 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001251 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001252 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001253 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001254};
1255
mflodmancc3d4422017-08-03 08:27:51 -07001256TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001257 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001258 DataRate::BitsPerSec(kTargetBitrateBps),
1259 DataRate::BitsPerSec(kTargetBitrateBps),
1260 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001261 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001262 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001263 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001264 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001265 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001266}
1267
mflodmancc3d4422017-08-03 08:27:51 -07001268TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001269 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001270 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001271 // The encoder will cache up to one frame for a short duration. Adding two
1272 // frames means that the first frame will be dropped and the second frame will
1273 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001274 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001275 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001276 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001277
Henrik Boström381d1092020-05-12 18:49:07 +02001278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001279 DataRate::BitsPerSec(kTargetBitrateBps),
1280 DataRate::BitsPerSec(kTargetBitrateBps),
1281 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001282
Sebastian Janssona3177052018-04-10 13:05:49 +02001283 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001284 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001285 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1286
1287 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001288 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001289}
1290
mflodmancc3d4422017-08-03 08:27:51 -07001291TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001293 DataRate::BitsPerSec(kTargetBitrateBps),
1294 DataRate::BitsPerSec(kTargetBitrateBps),
1295 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001296 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001297 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001298
Henrik Boström381d1092020-05-12 18:49:07 +02001299 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1300 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1301 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001302 // The encoder will cache up to one frame for a short duration. Adding two
1303 // frames means that the first frame will be dropped and the second frame will
1304 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001305 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001306 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001307
Henrik Boström381d1092020-05-12 18:49:07 +02001308 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001309 DataRate::BitsPerSec(kTargetBitrateBps),
1310 DataRate::BitsPerSec(kTargetBitrateBps),
1311 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001312 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001313 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1314 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001315 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001316}
1317
mflodmancc3d4422017-08-03 08:27:51 -07001318TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001320 DataRate::BitsPerSec(kTargetBitrateBps),
1321 DataRate::BitsPerSec(kTargetBitrateBps),
1322 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001323 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001324 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001325
1326 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001327 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001328
perkja49cbd32016-09-16 07:53:41 -07001329 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001330 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001331 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001332}
1333
mflodmancc3d4422017-08-03 08:27:51 -07001334TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001336 DataRate::BitsPerSec(kTargetBitrateBps),
1337 DataRate::BitsPerSec(kTargetBitrateBps),
1338 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001339
perkja49cbd32016-09-16 07:53:41 -07001340 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001341 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001342
mflodmancc3d4422017-08-03 08:27:51 -07001343 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001344 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001345 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001346 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1347 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001348}
1349
mflodmancc3d4422017-08-03 08:27:51 -07001350TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001352 DataRate::BitsPerSec(kTargetBitrateBps),
1353 DataRate::BitsPerSec(kTargetBitrateBps),
1354 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001355
1356 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001357 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001358 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001359 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1360 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001361 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1362 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001363 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001364 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001365
mflodmancc3d4422017-08-03 08:27:51 -07001366 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001367}
1368
Noah Richards51db4212019-06-12 06:59:12 -07001369TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001371 DataRate::BitsPerSec(kTargetBitrateBps),
1372 DataRate::BitsPerSec(kTargetBitrateBps),
1373 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001374
1375 rtc::Event frame_destroyed_event;
1376 video_source_.IncomingCapturedFrame(
1377 CreateFakeNativeFrame(1, &frame_destroyed_event));
1378 ExpectDroppedFrame();
1379 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1380 video_stream_encoder_->Stop();
1381}
1382
1383TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1384 // Use the cropping factory.
1385 video_encoder_config_.video_stream_factory =
1386 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1387 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1388 kMaxPayloadLength);
1389 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1390
1391 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001393 DataRate::BitsPerSec(kTargetBitrateBps),
1394 DataRate::BitsPerSec(kTargetBitrateBps),
1395 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001396 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1397 WaitForEncodedFrame(1);
1398 // The encoder will have been configured once.
1399 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1400 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1401 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1402
1403 // Now send in a fake frame that needs to be cropped as the width/height
1404 // aren't divisible by 4 (see CreateEncoderStreams above).
1405 rtc::Event frame_destroyed_event;
1406 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1407 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1408 ExpectDroppedFrame();
1409 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1410 video_stream_encoder_->Stop();
1411}
1412
Ying Wang9b881ab2020-02-07 14:29:32 +01001413TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001415 DataRate::BitsPerSec(kTargetBitrateBps),
1416 DataRate::BitsPerSec(kTargetBitrateBps),
1417 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001418 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1419 WaitForEncodedFrame(1);
1420
Henrik Boström381d1092020-05-12 18:49:07 +02001421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001422 DataRate::BitsPerSec(kTargetBitrateBps),
1423 DataRate::BitsPerSec(kTargetBitrateBps),
1424 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001425 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1426 // frames. Adding two frames means that the first frame will be dropped and
1427 // the second frame will be sent to the encoder.
1428 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1429 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1430 WaitForEncodedFrame(3);
1431 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1432 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1433 WaitForEncodedFrame(5);
1434 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1435 video_stream_encoder_->Stop();
1436}
1437
mflodmancc3d4422017-08-03 08:27:51 -07001438TEST_F(VideoStreamEncoderTest,
1439 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001440 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001441 DataRate::BitsPerSec(kTargetBitrateBps),
1442 DataRate::BitsPerSec(kTargetBitrateBps),
1443 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001444 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001445
1446 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001447 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001448 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001449 // The encoder will have been configured once when the first frame is
1450 // received.
1451 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001452
1453 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001454 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001455 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001456 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001457 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001458
1459 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001460 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001462 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001463 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001464
mflodmancc3d4422017-08-03 08:27:51 -07001465 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001466}
1467
mflodmancc3d4422017-08-03 08:27:51 -07001468TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001470 DataRate::BitsPerSec(kTargetBitrateBps),
1471 DataRate::BitsPerSec(kTargetBitrateBps),
1472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001473
1474 // Capture a frame and wait for it to synchronize with the encoder thread.
1475 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001476 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001477 // The encoder will have been configured once.
1478 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001479 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1480 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1481
1482 codec_width_ *= 2;
1483 codec_height_ *= 2;
1484 // Capture a frame with a higher resolution and wait for it to synchronize
1485 // with the encoder thread.
1486 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001487 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001488 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1489 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001490 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001491
mflodmancc3d4422017-08-03 08:27:51 -07001492 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001493}
1494
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001495TEST_F(VideoStreamEncoderTest,
1496 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001498 DataRate::BitsPerSec(kTargetBitrateBps),
1499 DataRate::BitsPerSec(kTargetBitrateBps),
1500 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001501
1502 // Capture a frame and wait for it to synchronize with the encoder thread.
1503 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1504 WaitForEncodedFrame(1);
1505
1506 VideoEncoderConfig video_encoder_config;
1507 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1508 // Changing the max payload data length recreates encoder.
1509 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1510 kMaxPayloadLength / 2);
1511
1512 // Capture a frame and wait for it to synchronize with the encoder thread.
1513 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1514 WaitForEncodedFrame(2);
1515 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1516
1517 video_stream_encoder_->Stop();
1518}
1519
Sergey Silkin5ee69672019-07-02 14:18:34 +02001520TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001521 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001522 DataRate::BitsPerSec(kTargetBitrateBps),
1523 DataRate::BitsPerSec(kTargetBitrateBps),
1524 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001525
1526 VideoEncoderConfig video_encoder_config;
1527 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1528 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1529 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1530 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1531 kMaxPayloadLength);
1532
1533 // Capture a frame and wait for it to synchronize with the encoder thread.
1534 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1535 WaitForEncodedFrame(1);
1536 // The encoder will have been configured once when the first frame is
1537 // received.
1538 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1539 EXPECT_EQ(kTargetBitrateBps,
1540 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1541 EXPECT_EQ(kStartBitrateBps,
1542 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1543
Sergey Silkin6456e352019-07-08 17:56:40 +02001544 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1545 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001546 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1547 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1548 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1549 kMaxPayloadLength);
1550
1551 // Capture a frame and wait for it to synchronize with the encoder thread.
1552 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1553 WaitForEncodedFrame(2);
1554 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1555 // Bitrate limits have changed - rate allocator should be reconfigured,
1556 // encoder should not be reconfigured.
1557 EXPECT_EQ(kTargetBitrateBps * 2,
1558 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1559 EXPECT_EQ(kStartBitrateBps * 2,
1560 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1561 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1562
1563 video_stream_encoder_->Stop();
1564}
1565
Sergey Silkin6456e352019-07-08 17:56:40 +02001566TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001567 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001569 DataRate::BitsPerSec(kTargetBitrateBps),
1570 DataRate::BitsPerSec(kTargetBitrateBps),
1571 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001572
Sergey Silkincd02eba2020-01-20 14:48:40 +01001573 const uint32_t kMinEncBitrateKbps = 100;
1574 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001575 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001576 /*frame_size_pixels=*/codec_width_ * codec_height_,
1577 /*min_start_bitrate_bps=*/0,
1578 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1579 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001580 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1581
Sergey Silkincd02eba2020-01-20 14:48:40 +01001582 VideoEncoderConfig video_encoder_config;
1583 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1584 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1585 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1586 (kMinEncBitrateKbps + 1) * 1000;
1587 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1588 kMaxPayloadLength);
1589
1590 // When both encoder and app provide bitrate limits, the intersection of
1591 // provided sets should be used.
1592 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1593 WaitForEncodedFrame(1);
1594 EXPECT_EQ(kMaxEncBitrateKbps,
1595 bitrate_allocator_factory_.codec_config().maxBitrate);
1596 EXPECT_EQ(kMinEncBitrateKbps + 1,
1597 bitrate_allocator_factory_.codec_config().minBitrate);
1598
1599 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1600 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1601 (kMinEncBitrateKbps - 1) * 1000;
1602 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1603 kMaxPayloadLength);
1604 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001605 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001606 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001607 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001608 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001609 bitrate_allocator_factory_.codec_config().minBitrate);
1610
Sergey Silkincd02eba2020-01-20 14:48:40 +01001611 video_stream_encoder_->Stop();
1612}
1613
1614TEST_F(VideoStreamEncoderTest,
1615 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001617 DataRate::BitsPerSec(kTargetBitrateBps),
1618 DataRate::BitsPerSec(kTargetBitrateBps),
1619 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001620
1621 const uint32_t kMinAppBitrateKbps = 100;
1622 const uint32_t kMaxAppBitrateKbps = 200;
1623 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1624 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1625 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1626 /*frame_size_pixels=*/codec_width_ * codec_height_,
1627 /*min_start_bitrate_bps=*/0,
1628 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1629 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1630 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1631
1632 VideoEncoderConfig video_encoder_config;
1633 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1634 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1635 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1636 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001637 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1638 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001639
Sergey Silkincd02eba2020-01-20 14:48:40 +01001640 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1641 WaitForEncodedFrame(1);
1642 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001643 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001644 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001645 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001646
1647 video_stream_encoder_->Stop();
1648}
1649
1650TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001651 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001653 DataRate::BitsPerSec(kTargetBitrateBps),
1654 DataRate::BitsPerSec(kTargetBitrateBps),
1655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001656
1657 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001658 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001659 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001660 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001661 fake_encoder_.SetResolutionBitrateLimits(
1662 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1663
1664 VideoEncoderConfig video_encoder_config;
1665 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1666 video_encoder_config.max_bitrate_bps = 0;
1667 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1668 kMaxPayloadLength);
1669
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001670 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001671 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1672 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001673 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1674 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001675 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1676 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1677
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001678 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001679 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1680 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001681 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1682 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001683 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1684 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1685
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001686 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001687 // encoder for 360p should be used.
1688 video_source_.IncomingCapturedFrame(
1689 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1690 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001691 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1692 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001693 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1694 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1695
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001696 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001697 // ignored.
1698 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1699 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001700 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1701 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001702 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1703 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001704 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1705 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001706 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1707 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1708
1709 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1710 // for 270p should be used.
1711 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1712 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001713 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1714 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001715 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1716 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1717
1718 video_stream_encoder_->Stop();
1719}
1720
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001721TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001722 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001723 DataRate::BitsPerSec(kTargetBitrateBps),
1724 DataRate::BitsPerSec(kTargetBitrateBps),
1725 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001726
1727 VideoEncoderConfig video_encoder_config;
1728 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1729 video_encoder_config.max_bitrate_bps = 0;
1730 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1731 kMaxPayloadLength);
1732
1733 // Encode 720p frame to get the default encoder target bitrate.
1734 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1735 WaitForEncodedFrame(1);
1736 const uint32_t kDefaultTargetBitrateFor720pKbps =
1737 bitrate_allocator_factory_.codec_config()
1738 .simulcastStream[0]
1739 .targetBitrate;
1740
1741 // Set the max recommended encoder bitrate to something lower than the default
1742 // target bitrate.
1743 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1744 1280 * 720, 10 * 1000, 10 * 1000,
1745 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1746 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1747
1748 // Change resolution to trigger encoder reinitialization.
1749 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1750 WaitForEncodedFrame(2);
1751 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1752 WaitForEncodedFrame(3);
1753
1754 // Ensure the target bitrate is capped by the max bitrate.
1755 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1756 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1757 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1758 .simulcastStream[0]
1759 .targetBitrate *
1760 1000,
1761 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1762
1763 video_stream_encoder_->Stop();
1764}
1765
mflodmancc3d4422017-08-03 08:27:51 -07001766TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001767 EXPECT_TRUE(video_source_.has_sinks());
1768 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001769 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001770 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001771 EXPECT_FALSE(video_source_.has_sinks());
1772 EXPECT_TRUE(new_video_source.has_sinks());
1773
mflodmancc3d4422017-08-03 08:27:51 -07001774 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001775}
1776
mflodmancc3d4422017-08-03 08:27:51 -07001777TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001778 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001779 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001780 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001781 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001782}
1783
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001784class ResolutionAlignmentTest
1785 : public VideoStreamEncoderTest,
1786 public ::testing::WithParamInterface<
1787 ::testing::tuple<int, std::vector<double>>> {
1788 public:
1789 ResolutionAlignmentTest()
1790 : requested_alignment_(::testing::get<0>(GetParam())),
1791 scale_factors_(::testing::get<1>(GetParam())) {}
1792
1793 protected:
1794 const int requested_alignment_;
1795 const std::vector<double> scale_factors_;
1796};
1797
1798INSTANTIATE_TEST_SUITE_P(
1799 AlignmentAndScaleFactors,
1800 ResolutionAlignmentTest,
1801 ::testing::Combine(
1802 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
1803 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
1804 std::vector<double>{-1.0, -1.0},
1805 std::vector<double>{-1.0, -1.0, -1.0},
1806 std::vector<double>{4.0, 2.0, 1.0},
1807 std::vector<double>{9999.0, -1.0, 1.0},
1808 std::vector<double>{3.99, 2.01, 1.0},
1809 std::vector<double>{4.9, 1.7, 1.25},
1810 std::vector<double>{10.0, 4.0, 3.0},
1811 std::vector<double>{1.75, 3.5},
1812 std::vector<double>{1.5, 2.5},
1813 std::vector<double>{1.3, 1.0})));
1814
1815TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
1816 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001817 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001818 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
1819 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
1820
1821 // Fill config with the scaling factor by which to reduce encoding size.
1822 const int num_streams = scale_factors_.size();
1823 VideoEncoderConfig config;
1824 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
1825 for (int i = 0; i < num_streams; ++i) {
1826 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
1827 }
1828 config.video_stream_factory =
1829 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
1830 "VP8", /*max qp*/ 56, /*screencast*/ false,
1831 /*screenshare enabled*/ false);
1832 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
1833
Henrik Boström381d1092020-05-12 18:49:07 +02001834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001835 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
1836 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
1837 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
1838 // Wait for all layers before triggering event.
1839 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001840
1841 // On the 1st frame, we should have initialized the encoder and
1842 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001843 int64_t timestamp_ms = kFrameIntervalMs;
1844 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
1845 WaitForEncodedFrame(timestamp_ms);
1846 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001847
1848 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1849 // (It's up the to the encoder to potentially drop the previous frame,
1850 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001851 timestamp_ms += kFrameIntervalMs;
1852 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
1853 WaitForEncodedFrame(timestamp_ms);
1854 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
1855
1856 VideoCodec codec = fake_encoder_.video_codec();
1857 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
1858 // Frame size should be a multiple of the requested alignment.
1859 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
1860 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
1861 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
1862 // Aspect ratio should match.
1863 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
1864 codec.height * codec.simulcastStream[i].width);
1865 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001866
1867 video_stream_encoder_->Stop();
1868}
1869
Jonathan Yubc771b72017-12-08 17:04:29 -08001870TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1871 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001872 const int kWidth = 1280;
1873 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001874
1875 // We rely on the automatic resolution adaptation, but we handle framerate
1876 // adaptation manually by mocking the stats proxy.
1877 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001878
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001879 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001881 DataRate::BitsPerSec(kTargetBitrateBps),
1882 DataRate::BitsPerSec(kTargetBitrateBps),
1883 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001884 video_stream_encoder_->SetSource(&video_source_,
1885 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001886 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001887 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001888 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001889 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1890
Jonathan Yubc771b72017-12-08 17:04:29 -08001891 // Adapt down as far as possible.
1892 rtc::VideoSinkWants last_wants;
1893 int64_t t = 1;
1894 int loop_count = 0;
1895 do {
1896 ++loop_count;
1897 last_wants = video_source_.sink_wants();
1898
1899 // Simulate the framerate we've been asked to adapt to.
1900 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1901 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1902 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1903 mock_stats.input_frame_rate = fps;
1904 stats_proxy_->SetMockStats(mock_stats);
1905
1906 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1907 sink_.WaitForEncodedFrame(t);
1908 t += frame_interval_ms;
1909
mflodmancc3d4422017-08-03 08:27:51 -07001910 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001911 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001912 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001913 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1914 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001915 } while (video_source_.sink_wants().max_pixel_count <
1916 last_wants.max_pixel_count ||
1917 video_source_.sink_wants().max_framerate_fps <
1918 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001919
Jonathan Yubc771b72017-12-08 17:04:29 -08001920 // Verify that we've adapted all the way down.
1921 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001922 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001923 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1924 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001925 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001926 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1927 *video_source_.last_sent_height());
1928 EXPECT_EQ(kMinBalancedFramerateFps,
1929 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001930
Jonathan Yubc771b72017-12-08 17:04:29 -08001931 // Adapt back up the same number of times we adapted down.
1932 for (int i = 0; i < loop_count - 1; ++i) {
1933 last_wants = video_source_.sink_wants();
1934
1935 // Simulate the framerate we've been asked to adapt to.
1936 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1937 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1938 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1939 mock_stats.input_frame_rate = fps;
1940 stats_proxy_->SetMockStats(mock_stats);
1941
1942 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1943 sink_.WaitForEncodedFrame(t);
1944 t += frame_interval_ms;
1945
Henrik Boström91aa7322020-04-28 12:24:33 +02001946 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001947 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001948 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001949 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1950 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001951 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1952 last_wants.max_pixel_count ||
1953 video_source_.sink_wants().max_framerate_fps >
1954 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001955 }
1956
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001957 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001958 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001959 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001960 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1961 EXPECT_EQ((loop_count - 1) * 2,
1962 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001963
mflodmancc3d4422017-08-03 08:27:51 -07001964 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001965}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001966
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001967TEST_F(VideoStreamEncoderTest,
1968 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1969 video_stream_encoder_->OnBitrateUpdated(
1970 DataRate::BitsPerSec(kTargetBitrateBps),
1971 DataRate::BitsPerSec(kTargetBitrateBps),
1972 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001973 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001974
1975 const int kFrameWidth = 1280;
1976 const int kFrameHeight = 720;
1977
1978 int64_t ntp_time = kFrameIntervalMs;
1979
1980 // Force an input frame rate to be available, or the adaptation call won't
1981 // know what framerate to adapt form.
1982 const int kInputFps = 30;
1983 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1984 stats.input_frame_rate = kInputFps;
1985 stats_proxy_->SetMockStats(stats);
1986
1987 video_source_.set_adaptation_enabled(true);
1988 video_stream_encoder_->SetSource(
1989 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001990 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001991 video_source_.IncomingCapturedFrame(
1992 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1993 sink_.WaitForEncodedFrame(ntp_time);
1994 ntp_time += kFrameIntervalMs;
1995
1996 // Trigger CPU overuse.
1997 video_stream_encoder_->TriggerCpuOveruse();
1998 video_source_.IncomingCapturedFrame(
1999 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2000 sink_.WaitForEncodedFrame(ntp_time);
2001 ntp_time += kFrameIntervalMs;
2002
2003 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2004 EXPECT_EQ(std::numeric_limits<int>::max(),
2005 video_source_.sink_wants().max_pixel_count);
2006 // Some framerate constraint should be set.
2007 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2008 EXPECT_LT(restricted_fps, kInputFps);
2009 video_source_.IncomingCapturedFrame(
2010 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2011 sink_.WaitForEncodedFrame(ntp_time);
2012 ntp_time += 100;
2013
Henrik Boström2671dac2020-05-19 16:29:09 +02002014 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002015 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2016 // Give the encoder queue time to process the change in degradation preference
2017 // by waiting for an encoded frame.
2018 video_source_.IncomingCapturedFrame(
2019 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2020 sink_.WaitForEncodedFrame(ntp_time);
2021 ntp_time += kFrameIntervalMs;
2022
2023 video_stream_encoder_->TriggerQualityLow();
2024 video_source_.IncomingCapturedFrame(
2025 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2026 sink_.WaitForEncodedFrame(ntp_time);
2027 ntp_time += kFrameIntervalMs;
2028
2029 // Some resolution constraint should be set.
2030 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2031 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2032 kFrameWidth * kFrameHeight);
2033 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2034
2035 int pixel_count = video_source_.sink_wants().max_pixel_count;
2036 // Triggering a CPU underuse should not change the sink wants since it has
2037 // not been overused for resolution since we changed degradation preference.
2038 video_stream_encoder_->TriggerCpuUnderuse();
2039 video_source_.IncomingCapturedFrame(
2040 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2041 sink_.WaitForEncodedFrame(ntp_time);
2042 ntp_time += kFrameIntervalMs;
2043 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2044 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2045
Evan Shrubsole64469032020-06-11 10:45:29 +02002046 // Change the degradation preference back. CPU underuse should not adapt since
2047 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002048 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002049 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2050 video_source_.IncomingCapturedFrame(
2051 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2052 sink_.WaitForEncodedFrame(ntp_time);
2053 ntp_time += 100;
2054 // Resolution adaptations is gone after changing degradation preference.
2055 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2056 EXPECT_EQ(std::numeric_limits<int>::max(),
2057 video_source_.sink_wants().max_pixel_count);
2058 // The fps adaptation from above is now back.
2059 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2060
2061 // Trigger CPU underuse.
2062 video_stream_encoder_->TriggerCpuUnderuse();
2063 video_source_.IncomingCapturedFrame(
2064 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2065 sink_.WaitForEncodedFrame(ntp_time);
2066 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002067 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2068
2069 // Trigger QP underuse, fps should return to normal.
2070 video_stream_encoder_->TriggerQualityHigh();
2071 video_source_.IncomingCapturedFrame(
2072 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2073 sink_.WaitForEncodedFrame(ntp_time);
2074 ntp_time += kFrameIntervalMs;
2075 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002076
2077 video_stream_encoder_->Stop();
2078}
2079
mflodmancc3d4422017-08-03 08:27:51 -07002080TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002081 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002082 DataRate::BitsPerSec(kTargetBitrateBps),
2083 DataRate::BitsPerSec(kTargetBitrateBps),
2084 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002085 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002086
sprangc5d62e22017-04-02 23:53:04 -07002087 const int kFrameWidth = 1280;
2088 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002089
Åsa Persson8c1bf952018-09-13 10:42:19 +02002090 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002091
kthelgason5e13d412016-12-01 03:59:51 -08002092 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002093 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002094 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002095 frame_timestamp += kFrameIntervalMs;
2096
perkj803d97f2016-11-01 11:45:46 -07002097 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002098 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002099 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002100 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002101 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002102 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002103
asapersson0944a802017-04-07 00:57:58 -07002104 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002105 // wanted resolution.
2106 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2107 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2108 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002109 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002110
2111 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002112 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002113 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002114 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002115 // Give the encoder queue time to process the change in degradation preference
2116 // by waiting for an encoded frame.
2117 new_video_source.IncomingCapturedFrame(
2118 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2119 sink_.WaitForEncodedFrame(frame_timestamp);
2120 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002121 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002122 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002123
sprangc5d62e22017-04-02 23:53:04 -07002124 // Force an input frame rate to be available, or the adaptation call won't
2125 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002126 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002127 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002128 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002129 stats_proxy_->SetMockStats(stats);
2130
mflodmancc3d4422017-08-03 08:27:51 -07002131 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002132 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002133 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002134 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002135 frame_timestamp += kFrameIntervalMs;
2136
2137 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002138 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002139 EXPECT_EQ(std::numeric_limits<int>::max(),
2140 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002141 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002142
asapersson02465b82017-04-10 01:12:52 -07002143 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002144 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2145 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002146 // Give the encoder queue time to process the change in degradation preference
2147 // by waiting for an encoded frame.
2148 new_video_source.IncomingCapturedFrame(
2149 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2150 sink_.WaitForEncodedFrame(frame_timestamp);
2151 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002152 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002153
mflodmancc3d4422017-08-03 08:27:51 -07002154 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002155 new_video_source.IncomingCapturedFrame(
2156 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002157 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002158 frame_timestamp += kFrameIntervalMs;
2159
2160 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002161 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002162
2163 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002164 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002165 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002166 // Give the encoder queue time to process the change in degradation preference
2167 // by waiting for an encoded frame.
2168 new_video_source.IncomingCapturedFrame(
2169 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2170 sink_.WaitForEncodedFrame(frame_timestamp);
2171 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002172 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2173 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002174 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002175 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002176
2177 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002178 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002179 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002180 // Give the encoder queue time to process the change in degradation preference
2181 // by waiting for an encoded frame.
2182 new_video_source.IncomingCapturedFrame(
2183 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2184 sink_.WaitForEncodedFrame(frame_timestamp);
2185 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002186 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2187 EXPECT_EQ(std::numeric_limits<int>::max(),
2188 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002189 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002190
mflodmancc3d4422017-08-03 08:27:51 -07002191 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002192}
2193
mflodmancc3d4422017-08-03 08:27:51 -07002194TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002196 DataRate::BitsPerSec(kTargetBitrateBps),
2197 DataRate::BitsPerSec(kTargetBitrateBps),
2198 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002199
asaperssonfab67072017-04-04 05:51:49 -07002200 const int kWidth = 1280;
2201 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002202 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002203 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002204 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2205 EXPECT_FALSE(stats.bw_limited_resolution);
2206 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2207
2208 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002209 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002210 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002211 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002212
2213 stats = stats_proxy_->GetStats();
2214 EXPECT_TRUE(stats.bw_limited_resolution);
2215 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2216
2217 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002218 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002219 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002220 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002221
2222 stats = stats_proxy_->GetStats();
2223 EXPECT_FALSE(stats.bw_limited_resolution);
2224 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2225 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2226
mflodmancc3d4422017-08-03 08:27:51 -07002227 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002228}
2229
mflodmancc3d4422017-08-03 08:27:51 -07002230TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002232 DataRate::BitsPerSec(kTargetBitrateBps),
2233 DataRate::BitsPerSec(kTargetBitrateBps),
2234 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002235
2236 const int kWidth = 1280;
2237 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002238 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002239 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002240 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2241 EXPECT_FALSE(stats.cpu_limited_resolution);
2242 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2243
2244 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002245 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002246 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002247 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002248
2249 stats = stats_proxy_->GetStats();
2250 EXPECT_TRUE(stats.cpu_limited_resolution);
2251 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2252
2253 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002254 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002255 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002256 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002257
2258 stats = stats_proxy_->GetStats();
2259 EXPECT_FALSE(stats.cpu_limited_resolution);
2260 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002261 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002262
mflodmancc3d4422017-08-03 08:27:51 -07002263 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002264}
2265
mflodmancc3d4422017-08-03 08:27:51 -07002266TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002268 DataRate::BitsPerSec(kTargetBitrateBps),
2269 DataRate::BitsPerSec(kTargetBitrateBps),
2270 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002271
asaperssonfab67072017-04-04 05:51:49 -07002272 const int kWidth = 1280;
2273 const int kHeight = 720;
2274 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002275 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002276 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002277 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002278 EXPECT_FALSE(stats.cpu_limited_resolution);
2279 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2280
asaperssonfab67072017-04-04 05:51:49 -07002281 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002282 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002283 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002284 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002285 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002286 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002287 EXPECT_TRUE(stats.cpu_limited_resolution);
2288 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2289
2290 // Set new source with adaptation still enabled.
2291 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002292 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002293 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002294
asaperssonfab67072017-04-04 05:51:49 -07002295 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002296 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002297 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002298 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002299 EXPECT_TRUE(stats.cpu_limited_resolution);
2300 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2301
2302 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002303 video_stream_encoder_->SetSource(&new_video_source,
2304 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002305
asaperssonfab67072017-04-04 05:51:49 -07002306 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002307 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002308 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002309 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002310 EXPECT_FALSE(stats.cpu_limited_resolution);
2311 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2312
2313 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002314 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002315 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002316
asaperssonfab67072017-04-04 05:51:49 -07002317 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002318 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002319 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002320 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002321 EXPECT_TRUE(stats.cpu_limited_resolution);
2322 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2323
asaperssonfab67072017-04-04 05:51:49 -07002324 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002325 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002326 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002327 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002328 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002329 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002330 EXPECT_FALSE(stats.cpu_limited_resolution);
2331 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002332 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002333
mflodmancc3d4422017-08-03 08:27:51 -07002334 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002335}
2336
mflodmancc3d4422017-08-03 08:27:51 -07002337TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002339 DataRate::BitsPerSec(kTargetBitrateBps),
2340 DataRate::BitsPerSec(kTargetBitrateBps),
2341 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002342
asaperssonfab67072017-04-04 05:51:49 -07002343 const int kWidth = 1280;
2344 const int kHeight = 720;
2345 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002346 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002347 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002348 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002349 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002350 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002351
2352 // Set new source with adaptation still enabled.
2353 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002354 video_stream_encoder_->SetSource(&new_video_source,
2355 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002356
asaperssonfab67072017-04-04 05:51:49 -07002357 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002358 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002359 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002360 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002361 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002362 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002363
asaperssonfab67072017-04-04 05:51:49 -07002364 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002365 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002366 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002367 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002368 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002369 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002370 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002371 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002372
asaperssonfab67072017-04-04 05:51:49 -07002373 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002374 video_stream_encoder_->SetSource(&new_video_source,
2375 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002376
asaperssonfab67072017-04-04 05:51:49 -07002377 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002378 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002379 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002380 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002381 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002382 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002383
asapersson02465b82017-04-10 01:12:52 -07002384 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002385 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002386 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002387
asaperssonfab67072017-04-04 05:51:49 -07002388 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002389 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002390 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002391 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002392 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002393 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2394 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002395
mflodmancc3d4422017-08-03 08:27:51 -07002396 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002397}
2398
mflodmancc3d4422017-08-03 08:27:51 -07002399TEST_F(VideoStreamEncoderTest,
2400 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002401 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002402 DataRate::BitsPerSec(kTargetBitrateBps),
2403 DataRate::BitsPerSec(kTargetBitrateBps),
2404 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002405
2406 const int kWidth = 1280;
2407 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002408 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002409 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002410 video_source_.IncomingCapturedFrame(
2411 CreateFrame(timestamp_ms, kWidth, kHeight));
2412 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002413 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2415 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2416
2417 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002418 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002419 timestamp_ms += kFrameIntervalMs;
2420 video_source_.IncomingCapturedFrame(
2421 CreateFrame(timestamp_ms, kWidth, kHeight));
2422 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002423 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2425 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2426
2427 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002428 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002429 timestamp_ms += kFrameIntervalMs;
2430 video_source_.IncomingCapturedFrame(
2431 CreateFrame(timestamp_ms, kWidth, kHeight));
2432 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002433 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2435 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2436
Niels Möller4db138e2018-04-19 09:04:13 +02002437 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002438 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002439
2440 VideoEncoderConfig video_encoder_config;
2441 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2442 // Make format different, to force recreation of encoder.
2443 video_encoder_config.video_format.parameters["foo"] = "foo";
2444 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002445 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002446 timestamp_ms += kFrameIntervalMs;
2447 video_source_.IncomingCapturedFrame(
2448 CreateFrame(timestamp_ms, kWidth, kHeight));
2449 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002450 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2452 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2453
mflodmancc3d4422017-08-03 08:27:51 -07002454 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002455}
2456
mflodmancc3d4422017-08-03 08:27:51 -07002457TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002458 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002459 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002460 DataRate::BitsPerSec(kTargetBitrateBps),
2461 DataRate::BitsPerSec(kTargetBitrateBps),
2462 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2463
2464 const int kWidth = 1280;
2465 const int kHeight = 720;
2466 int sequence = 1;
2467
2468 // Enable BALANCED preference, no initial limitation.
2469 test::FrameForwarder source;
2470 video_stream_encoder_->SetSource(&source,
2471 webrtc::DegradationPreference::BALANCED);
2472 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2473 WaitForEncodedFrame(sequence++);
2474 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2475 EXPECT_FALSE(stats.cpu_limited_resolution);
2476 EXPECT_FALSE(stats.cpu_limited_framerate);
2477 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2478
2479 // Trigger CPU overuse, should now adapt down.
2480 video_stream_encoder_->TriggerCpuOveruse();
2481 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2482 WaitForEncodedFrame(sequence++);
2483 stats = stats_proxy_->GetStats();
2484 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2485
2486 // Set new degradation preference should clear restrictions since we changed
2487 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002488 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002489 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2490 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2491 WaitForEncodedFrame(sequence++);
2492 stats = stats_proxy_->GetStats();
2493 EXPECT_FALSE(stats.cpu_limited_resolution);
2494 EXPECT_FALSE(stats.cpu_limited_framerate);
2495 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2496
2497 // Force an input frame rate to be available, or the adaptation call won't
2498 // know what framerate to adapt from.
2499 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2500 mock_stats.input_frame_rate = 30;
2501 stats_proxy_->SetMockStats(mock_stats);
2502 video_stream_encoder_->TriggerCpuOveruse();
2503 stats_proxy_->ResetMockStats();
2504 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2505 WaitForEncodedFrame(sequence++);
2506
2507 // We have now adapted once.
2508 stats = stats_proxy_->GetStats();
2509 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2510
2511 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002512 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2513 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002514 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2515 WaitForEncodedFrame(sequence++);
2516 stats = stats_proxy_->GetStats();
2517 EXPECT_FALSE(stats.cpu_limited_resolution);
2518 EXPECT_FALSE(stats.cpu_limited_framerate);
2519 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2520
2521 video_stream_encoder_->Stop();
2522}
2523
2524TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002525 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002527 DataRate::BitsPerSec(kTargetBitrateBps),
2528 DataRate::BitsPerSec(kTargetBitrateBps),
2529 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002530
asapersson0944a802017-04-07 00:57:58 -07002531 const int kWidth = 1280;
2532 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002533 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002534
asaperssonfab67072017-04-04 05:51:49 -07002535 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002536 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002537 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002538 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002539 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002540 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2541
asapersson02465b82017-04-10 01:12:52 -07002542 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002543 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002544 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002545 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002546 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002547 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002548 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002549 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2550
2551 // Set new source with adaptation still enabled.
2552 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002553 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002554 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002555
2556 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002557 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002558 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002559 stats = stats_proxy_->GetStats();
2560 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002561 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002562 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2563
sprangc5d62e22017-04-02 23:53:04 -07002564 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002565 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002566 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002567 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002568 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002569 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002570 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002571 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002572 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002573 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002574 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2575
sprangc5d62e22017-04-02 23:53:04 -07002576 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002577 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002578 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2579 mock_stats.input_frame_rate = 30;
2580 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002581 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002582 stats_proxy_->ResetMockStats();
2583
2584 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002585 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002586 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002587
2588 // Framerate now adapted.
2589 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002590 EXPECT_FALSE(stats.cpu_limited_resolution);
2591 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002592 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2593
2594 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002595 video_stream_encoder_->SetSource(&new_video_source,
2596 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002597 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002598 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002599 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002600
2601 stats = stats_proxy_->GetStats();
2602 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002603 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002604 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2605
2606 // Try to trigger overuse. Should not succeed.
2607 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002608 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002609 stats_proxy_->ResetMockStats();
2610
2611 stats = stats_proxy_->GetStats();
2612 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002613 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002614 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2615
2616 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002617 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002618 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002619 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002620 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002621 stats = stats_proxy_->GetStats();
2622 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002623 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002624 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002625
2626 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002627 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002628 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002629 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002630 stats = stats_proxy_->GetStats();
2631 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002632 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002633 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2634
2635 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002636 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002637 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002638 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002639 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002640 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002641 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002642 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002643 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002644 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002645 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2646
2647 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002648 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002649 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002650 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002651 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002652 stats = stats_proxy_->GetStats();
2653 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002654 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002655 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002656 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002657
mflodmancc3d4422017-08-03 08:27:51 -07002658 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002659}
2660
mflodmancc3d4422017-08-03 08:27:51 -07002661TEST_F(VideoStreamEncoderTest,
2662 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002663 const int kWidth = 1280;
2664 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002666 DataRate::BitsPerSec(kTargetBitrateBps),
2667 DataRate::BitsPerSec(kTargetBitrateBps),
2668 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002669
asaperssonfab67072017-04-04 05:51:49 -07002670 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002671 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002672
asaperssonfab67072017-04-04 05:51:49 -07002673 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002674 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002675
asaperssonfab67072017-04-04 05:51:49 -07002676 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002677 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002678
asaperssonfab67072017-04-04 05:51:49 -07002679 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002680 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002681
kthelgason876222f2016-11-29 01:44:11 -08002682 // Expect a scale down.
2683 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002684 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002685
asapersson02465b82017-04-10 01:12:52 -07002686 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002687 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002689 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002690
asaperssonfab67072017-04-04 05:51:49 -07002691 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002692 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002693 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002694 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002695
asaperssonfab67072017-04-04 05:51:49 -07002696 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002697 EXPECT_EQ(std::numeric_limits<int>::max(),
2698 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002699
asaperssonfab67072017-04-04 05:51:49 -07002700 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002701 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002702 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002703 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002704
asapersson02465b82017-04-10 01:12:52 -07002705 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002706 EXPECT_EQ(std::numeric_limits<int>::max(),
2707 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002708
mflodmancc3d4422017-08-03 08:27:51 -07002709 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002710}
2711
mflodmancc3d4422017-08-03 08:27:51 -07002712TEST_F(VideoStreamEncoderTest,
2713 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002714 const int kWidth = 1280;
2715 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002717 DataRate::BitsPerSec(kTargetBitrateBps),
2718 DataRate::BitsPerSec(kTargetBitrateBps),
2719 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002720
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002721 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002722 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002723 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002724 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002725
2726 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002727 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002728 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2731
2732 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002733 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002734 EXPECT_THAT(source.sink_wants(),
2735 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002736 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2737 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2738 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2739
2740 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002741 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002742 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2743 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2744 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2745
mflodmancc3d4422017-08-03 08:27:51 -07002746 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002747}
2748
mflodmancc3d4422017-08-03 08:27:51 -07002749TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002750 const int kWidth = 1280;
2751 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002753 DataRate::BitsPerSec(kTargetBitrateBps),
2754 DataRate::BitsPerSec(kTargetBitrateBps),
2755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002756
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002757 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002758 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002759 video_stream_encoder_->SetSource(&source,
2760 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002761 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2762 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002763 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002764
2765 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002766 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002767 EXPECT_THAT(source.sink_wants(),
2768 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002769 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2770 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2771 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2772
2773 // Trigger adapt down for same input resolution, expect no change.
2774 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2775 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002776 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002777 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2778 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2779 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2780
2781 // Trigger adapt down for larger input resolution, expect no change.
2782 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2783 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002784 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002785 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2786 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2787 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2788
mflodmancc3d4422017-08-03 08:27:51 -07002789 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002790}
2791
mflodmancc3d4422017-08-03 08:27:51 -07002792TEST_F(VideoStreamEncoderTest,
2793 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002794 const int kWidth = 1280;
2795 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002797 DataRate::BitsPerSec(kTargetBitrateBps),
2798 DataRate::BitsPerSec(kTargetBitrateBps),
2799 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002800
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002801 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002802 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002803 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002804 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002805
2806 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002807 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002808 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2810 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2811
2812 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002813 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2816 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2817
mflodmancc3d4422017-08-03 08:27:51 -07002818 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002819}
2820
mflodmancc3d4422017-08-03 08:27:51 -07002821TEST_F(VideoStreamEncoderTest,
2822 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002823 const int kWidth = 1280;
2824 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002826 DataRate::BitsPerSec(kTargetBitrateBps),
2827 DataRate::BitsPerSec(kTargetBitrateBps),
2828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002829
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002830 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002831 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002832 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002833 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002834
2835 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002836 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002837 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002838 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002839 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2840
2841 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002842 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002843 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2846
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002848}
2849
mflodmancc3d4422017-08-03 08:27:51 -07002850TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002851 const int kWidth = 1280;
2852 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002854 DataRate::BitsPerSec(kTargetBitrateBps),
2855 DataRate::BitsPerSec(kTargetBitrateBps),
2856 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002857
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002858 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002859 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002860 video_stream_encoder_->SetSource(&source,
2861 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002862
2863 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2864 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002865 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002866 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2868 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2869
2870 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002871 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002872 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002873 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2874 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2875 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2876
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002878}
2879
mflodmancc3d4422017-08-03 08:27:51 -07002880TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002881 const int kWidth = 1280;
2882 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002883 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002884 DataRate::BitsPerSec(kTargetBitrateBps),
2885 DataRate::BitsPerSec(kTargetBitrateBps),
2886 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002887
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002888 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002889 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002890 video_stream_encoder_->SetSource(&source,
2891 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002892
2893 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2894 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002895 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2897 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2898 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2899
2900 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002901 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002902 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2904 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2905 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2906
mflodmancc3d4422017-08-03 08:27:51 -07002907 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002908}
2909
mflodmancc3d4422017-08-03 08:27:51 -07002910TEST_F(VideoStreamEncoderTest,
2911 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002912 const int kWidth = 1280;
2913 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002914 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002915 DataRate::BitsPerSec(kTargetBitrateBps),
2916 DataRate::BitsPerSec(kTargetBitrateBps),
2917 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002918
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002919 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002920 AdaptingFrameForwarder source;
2921 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002922 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002923 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002924
2925 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002926 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002927 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002928 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2929 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2930
2931 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002932 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002933 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002934 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002935 EXPECT_THAT(source.sink_wants(),
2936 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002937 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2938 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2939
2940 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002941 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002942 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002943 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2944 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2945 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2946
mflodmancc3d4422017-08-03 08:27:51 -07002947 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002948}
2949
mflodmancc3d4422017-08-03 08:27:51 -07002950TEST_F(VideoStreamEncoderTest,
2951 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002952 const int kWidth = 1280;
2953 const int kHeight = 720;
2954 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002955 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002956 DataRate::BitsPerSec(kTargetBitrateBps),
2957 DataRate::BitsPerSec(kTargetBitrateBps),
2958 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002959
2960 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2961 stats.input_frame_rate = kInputFps;
2962 stats_proxy_->SetMockStats(stats);
2963
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002964 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002965 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2966 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002967 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002968
2969 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002970 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002971 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2972 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002973 EXPECT_THAT(video_source_.sink_wants(),
2974 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002975
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002976 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002977 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002978 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002979 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002980 // Give the encoder queue time to process the change in degradation preference
2981 // by waiting for an encoded frame.
2982 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2983 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002984 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002985
2986 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002987 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002988 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2989 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002990 EXPECT_THAT(new_video_source.sink_wants(),
2991 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002992
2993 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002994 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002995 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002996
mflodmancc3d4422017-08-03 08:27:51 -07002997 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002998}
2999
mflodmancc3d4422017-08-03 08:27:51 -07003000TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003001 const int kWidth = 1280;
3002 const int kHeight = 720;
3003 const size_t kNumFrames = 10;
3004
Henrik Boström381d1092020-05-12 18:49:07 +02003005 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003006 DataRate::BitsPerSec(kTargetBitrateBps),
3007 DataRate::BitsPerSec(kTargetBitrateBps),
3008 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003009
asaperssond0de2952017-04-21 01:47:31 -07003010 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003011 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003012 video_source_.set_adaptation_enabled(true);
3013
3014 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3015 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3016
3017 int downscales = 0;
3018 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003019 video_source_.IncomingCapturedFrame(
3020 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3021 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003022
asaperssonfab67072017-04-04 05:51:49 -07003023 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003024 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003025 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003026 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003027
3028 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3029 ++downscales;
3030
3031 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3032 EXPECT_EQ(downscales,
3033 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3034 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003035 }
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003037}
3038
mflodmancc3d4422017-08-03 08:27:51 -07003039TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003040 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3041 const int kWidth = 1280;
3042 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003044 DataRate::BitsPerSec(kTargetBitrateBps),
3045 DataRate::BitsPerSec(kTargetBitrateBps),
3046 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003047
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003048 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003049 AdaptingFrameForwarder source;
3050 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003051 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003052 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003053
Åsa Persson8c1bf952018-09-13 10:42:19 +02003054 int64_t timestamp_ms = kFrameIntervalMs;
3055 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003056 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003057 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003058 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3059 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3060
3061 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003062 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003063 timestamp_ms += kFrameIntervalMs;
3064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3065 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003066 EXPECT_THAT(source.sink_wants(),
3067 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003068 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3069 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3070
3071 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003072 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003073 timestamp_ms += kFrameIntervalMs;
3074 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003075 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003076 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003077 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3078 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3079
3080 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003081 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003082 timestamp_ms += kFrameIntervalMs;
3083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3084 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003085 EXPECT_THAT(source.sink_wants(),
3086 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003087 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3088 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3089
3090 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003091 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003092 timestamp_ms += kFrameIntervalMs;
3093 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003094 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003095 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003096 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3097 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3098
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003100}
3101
mflodmancc3d4422017-08-03 08:27:51 -07003102TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003103 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3104 const int kWidth = 1280;
3105 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003107 DataRate::BitsPerSec(kTargetBitrateBps),
3108 DataRate::BitsPerSec(kTargetBitrateBps),
3109 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003110
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003111 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003112 AdaptingFrameForwarder source;
3113 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003114 video_stream_encoder_->SetSource(&source,
3115 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003116
Åsa Persson8c1bf952018-09-13 10:42:19 +02003117 int64_t timestamp_ms = kFrameIntervalMs;
3118 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003119 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003120 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003121 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3122 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3123
3124 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003125 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003126 timestamp_ms += kFrameIntervalMs;
3127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3128 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003129 EXPECT_THAT(source.sink_wants(),
3130 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003131 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3132 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3133
3134 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003135 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003136 timestamp_ms += kFrameIntervalMs;
3137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003138 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003139 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003140 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3141 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3142
3143 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003144 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003145 timestamp_ms += kFrameIntervalMs;
3146 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3147 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003148 EXPECT_THAT(source.sink_wants(),
3149 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003150 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3151 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3152
3153 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003154 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003155 timestamp_ms += kFrameIntervalMs;
3156 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003157 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003158 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003159 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3160 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3161
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003163}
3164
Sergey Silkin41c650b2019-10-14 13:12:19 +02003165TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3166 fake_encoder_.SetResolutionBitrateLimits(
3167 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3168
Henrik Boström381d1092020-05-12 18:49:07 +02003169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003170 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3171 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3172 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3173 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003174
3175 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3176 AdaptingFrameForwarder source;
3177 source.set_adaptation_enabled(true);
3178 video_stream_encoder_->SetSource(
3179 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3180
3181 // Insert 720p frame.
3182 int64_t timestamp_ms = kFrameIntervalMs;
3183 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3184 WaitForEncodedFrame(1280, 720);
3185
3186 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003187 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003188 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3189 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3190 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3191 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003192 video_stream_encoder_->TriggerQualityLow();
3193
3194 // Insert 720p frame. It should be downscaled and encoded.
3195 timestamp_ms += kFrameIntervalMs;
3196 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3197 WaitForEncodedFrame(960, 540);
3198
3199 // Trigger adapt up. Higher resolution should not be requested duo to lack
3200 // of bitrate.
3201 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003202 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003203
3204 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003206 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3207 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3208 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3209 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003210
3211 // Trigger adapt up. Higher resolution should be requested.
3212 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003213 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003214
3215 video_stream_encoder_->Stop();
3216}
3217
3218TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3219 fake_encoder_.SetResolutionBitrateLimits(
3220 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3221
3222 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003224 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3225 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3226 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3227 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003228
3229 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3230 AdaptingFrameForwarder source;
3231 source.set_adaptation_enabled(true);
3232 video_stream_encoder_->SetSource(
3233 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3234
3235 // Insert 720p frame. It should be dropped and lower resolution should be
3236 // requested.
3237 int64_t timestamp_ms = kFrameIntervalMs;
3238 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3239 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003240 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003241
3242 // Insert 720p frame. It should be downscaled and encoded.
3243 timestamp_ms += kFrameIntervalMs;
3244 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3245 WaitForEncodedFrame(960, 540);
3246
3247 video_stream_encoder_->Stop();
3248}
3249
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003250class BalancedDegradationTest : public VideoStreamEncoderTest {
3251 protected:
3252 void SetupTest() {
3253 // Reset encoder for field trials to take effect.
3254 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003255 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003256
3257 // Enable BALANCED preference.
3258 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003259 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3260 }
3261
3262 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003263 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003264 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3265 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003266 }
3267
Åsa Persson45b176f2019-09-30 11:19:05 +02003268 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003269 timestamp_ms_ += kFrameIntervalMs;
3270 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003271 }
3272
3273 void InsertFrameAndWaitForEncoded() {
3274 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003275 sink_.WaitForEncodedFrame(timestamp_ms_);
3276 }
3277
3278 const int kWidth = 640; // pixels:640x360=230400
3279 const int kHeight = 360;
3280 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3281 int64_t timestamp_ms_ = 0;
3282 AdaptingFrameForwarder source_;
3283};
3284
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003285TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003286 test::ScopedFieldTrials field_trials(
3287 "WebRTC-Video-BalancedDegradationSettings/"
3288 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3289 SetupTest();
3290
3291 // Force input frame rate.
3292 const int kInputFps = 24;
3293 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3294 stats.input_frame_rate = kInputFps;
3295 stats_proxy_->SetMockStats(stats);
3296
Åsa Persson45b176f2019-09-30 11:19:05 +02003297 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003298 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003299
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003300 // Trigger adapt down, expect scaled down framerate and resolution,
3301 // since Fps diff (input-requested:0) < threshold.
3302 video_stream_encoder_->TriggerQualityLow();
3303 EXPECT_THAT(source_.sink_wants(),
3304 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003305
3306 video_stream_encoder_->Stop();
3307}
3308
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003309TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003310 test::ScopedFieldTrials field_trials(
3311 "WebRTC-Video-BalancedDegradationSettings/"
3312 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3313 SetupTest();
3314
3315 // Force input frame rate.
3316 const int kInputFps = 25;
3317 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3318 stats.input_frame_rate = kInputFps;
3319 stats_proxy_->SetMockStats(stats);
3320
Åsa Persson45b176f2019-09-30 11:19:05 +02003321 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003323
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003324 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3325 // Fps diff (input-requested:1) == threshold.
3326 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003327 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003328
3329 video_stream_encoder_->Stop();
3330}
3331
3332TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3333 test::ScopedFieldTrials field_trials(
3334 "WebRTC-Video-BalancedDegradationSettings/"
3335 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3336 SetupTest();
3337
3338 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3339
Åsa Persson45b176f2019-09-30 11:19:05 +02003340 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003341 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003342
3343 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3344 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003345 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003346
3347 video_stream_encoder_->Stop();
3348}
3349
Åsa Perssonccfb3402019-09-25 15:13:04 +02003350TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003351 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003352 "WebRTC-Video-BalancedDegradationSettings/"
3353 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003354 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003355
Åsa Persson1b247f12019-08-14 17:26:39 +02003356 const int kMinBitrateBps = 425000;
3357 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003358 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003359
Åsa Persson45b176f2019-09-30 11:19:05 +02003360 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003361 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003362 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3363
3364 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3365 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003366 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003367 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003368 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3369
3370 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3371 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003372 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003373 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003374 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3375
Åsa Persson30ab0152019-08-27 12:22:33 +02003376 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3377 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003378 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003379 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003380 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003381 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003384 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003385 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003386 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003387
Åsa Persson30ab0152019-08-27 12:22:33 +02003388 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003389 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003390 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003391 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003392 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003393 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3394
3395 video_stream_encoder_->Stop();
3396}
3397
Åsa Perssonccfb3402019-09-25 15:13:04 +02003398TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003399 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3400 test::ScopedFieldTrials field_trials(
3401 "WebRTC-Video-BalancedDegradationSettings/"
3402 "pixels:57600|129600|230400,fps:7|24|24/");
3403 SetupTest();
3404 OnBitrateUpdated(kLowTargetBitrateBps);
3405
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003406 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003407
3408 // Insert frame, expect scaled down:
3409 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3410 InsertFrame();
3411 EXPECT_FALSE(WaitForFrame(1000));
3412 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3413 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3414
3415 // Insert frame, expect scaled down:
3416 // resolution (320x180@24fps).
3417 InsertFrame();
3418 EXPECT_FALSE(WaitForFrame(1000));
3419 EXPECT_LT(source_.sink_wants().max_pixel_count,
3420 source_.last_wants().max_pixel_count);
3421 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3422
3423 // Frame should not be dropped (min pixels per frame reached).
3424 InsertFrameAndWaitForEncoded();
3425
3426 video_stream_encoder_->Stop();
3427}
3428
3429TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003430 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003431 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003432 "WebRTC-Video-BalancedDegradationSettings/"
3433 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003434 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003435
Åsa Persson30ab0152019-08-27 12:22:33 +02003436 const int kResolutionMinBitrateBps = 435000;
3437 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003438 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003439
Åsa Persson45b176f2019-09-30 11:19:05 +02003440 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003441 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003442 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3443
3444 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3445 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003446 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003447 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003448 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3449
3450 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3451 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003452 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003453 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003454 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3455
3456 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3457 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003458 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003459 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003460 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3461
Åsa Persson30ab0152019-08-27 12:22:33 +02003462 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3463 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003464 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003465 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003466 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3467
3468 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3469 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003470 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003471 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003474 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003475 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003476 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003477 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003478 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3479
3480 video_stream_encoder_->Stop();
3481}
3482
Åsa Perssonccfb3402019-09-25 15:13:04 +02003483TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003484 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003485 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003486 "WebRTC-Video-BalancedDegradationSettings/"
3487 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003488 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003489
Åsa Persson30ab0152019-08-27 12:22:33 +02003490 const int kMinBitrateBps = 425000;
3491 const int kTooLowMinBitrateBps = 424000;
3492 const int kResolutionMinBitrateBps = 435000;
3493 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003494 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003495
Åsa Persson45b176f2019-09-30 11:19:05 +02003496 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003497 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003498 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3499
3500 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3501 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003502 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003503 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003504 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3505
3506 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3507 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003508 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003509 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003510 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3511
3512 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3513 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003514 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003515 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003516 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3517
3518 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3519 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003520 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003521 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3522
3523 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003524 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003525 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003526 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003527 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003528 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3529
3530 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003531 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003532 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003533 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003534 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3535
3536 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003537 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003538 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003539 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003540 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003541 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3542
Åsa Persson1b247f12019-08-14 17:26:39 +02003543 video_stream_encoder_->Stop();
3544}
3545
mflodmancc3d4422017-08-03 08:27:51 -07003546TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003547 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3548 const int kWidth = 1280;
3549 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003551 DataRate::BitsPerSec(kTargetBitrateBps),
3552 DataRate::BitsPerSec(kTargetBitrateBps),
3553 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003554
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003555 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003556 AdaptingFrameForwarder source;
3557 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003558 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003559 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003560
Åsa Persson8c1bf952018-09-13 10:42:19 +02003561 int64_t timestamp_ms = kFrameIntervalMs;
3562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003563 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003565 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3567 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3568 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3569
3570 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003571 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003572 timestamp_ms += kFrameIntervalMs;
3573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3574 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003575 EXPECT_THAT(source.sink_wants(),
3576 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003577 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3578 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3579 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3580 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3581
3582 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003583 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003584 timestamp_ms += kFrameIntervalMs;
3585 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3586 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003587 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003588 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3589 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3590 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3591 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3592
Jonathan Yubc771b72017-12-08 17:04:29 -08003593 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003594 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003595 timestamp_ms += kFrameIntervalMs;
3596 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3597 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003598 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003599 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3600 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003601 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003602 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3603
Jonathan Yubc771b72017-12-08 17:04:29 -08003604 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003605 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003606 timestamp_ms += kFrameIntervalMs;
3607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3608 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003609 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003610 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003611 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3613 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3614 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615
Jonathan Yubc771b72017-12-08 17:04:29 -08003616 // Trigger quality adapt down, expect no change (min resolution reached).
3617 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003618 timestamp_ms += kFrameIntervalMs;
3619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3620 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003621 EXPECT_THAT(source.sink_wants(), FpsMax());
3622 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003623 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3624 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3625 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3626 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3627
Evan Shrubsole64469032020-06-11 10:45:29 +02003628 // Trigger quality adapt up, expect upscaled resolution (480x270).
3629 video_stream_encoder_->TriggerQualityHigh();
3630 timestamp_ms += kFrameIntervalMs;
3631 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3632 WaitForEncodedFrame(timestamp_ms);
3633 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3634 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3636 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3637 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3638
3639 // Trigger quality and cpu adapt up since both are most limited, expect
3640 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003641 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003642 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003643 timestamp_ms += kFrameIntervalMs;
3644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3645 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003646 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003647 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3649 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003650 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003651
Evan Shrubsole64469032020-06-11 10:45:29 +02003652 // Trigger quality and cpu adapt up since both are most limited, expect
3653 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003654 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003655 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003656 timestamp_ms += kFrameIntervalMs;
3657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3658 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003659 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003660 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003661 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003662 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003663 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3664 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003665
Evan Shrubsole64469032020-06-11 10:45:29 +02003666 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3667 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003668 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003669 timestamp_ms += kFrameIntervalMs;
3670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3671 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003672 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003673 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3674 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003675 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003676 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003677
3678 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003679 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003680 timestamp_ms += kFrameIntervalMs;
3681 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003682 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003684 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003685 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3686 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003687 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003688 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003689
mflodmancc3d4422017-08-03 08:27:51 -07003690 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003691}
3692
mflodmancc3d4422017-08-03 08:27:51 -07003693TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003694 const int kWidth = 640;
3695 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003696
Henrik Boström381d1092020-05-12 18:49:07 +02003697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003698 DataRate::BitsPerSec(kTargetBitrateBps),
3699 DataRate::BitsPerSec(kTargetBitrateBps),
3700 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003701
perkj803d97f2016-11-01 11:45:46 -07003702 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003703 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003704 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003705 }
3706
mflodmancc3d4422017-08-03 08:27:51 -07003707 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003708 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003709 video_source_.IncomingCapturedFrame(CreateFrame(
3710 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003711 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003712 }
3713
mflodmancc3d4422017-08-03 08:27:51 -07003714 video_stream_encoder_->Stop();
3715 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003716 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003717
Ying Wangef3998f2019-12-09 13:06:53 +01003718 EXPECT_METRIC_EQ(
3719 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3720 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003721 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3722}
3723
mflodmancc3d4422017-08-03 08:27:51 -07003724TEST_F(VideoStreamEncoderTest,
3725 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003727 DataRate::BitsPerSec(kTargetBitrateBps),
3728 DataRate::BitsPerSec(kTargetBitrateBps),
3729 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003730 const int kWidth = 640;
3731 const int kHeight = 360;
3732
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003733 video_stream_encoder_->SetSource(&video_source_,
3734 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003735
3736 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3737 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003738 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003739 }
3740
mflodmancc3d4422017-08-03 08:27:51 -07003741 video_stream_encoder_->Stop();
3742 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003743 stats_proxy_.reset();
3744
3745 EXPECT_EQ(0,
3746 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3747}
3748
mflodmancc3d4422017-08-03 08:27:51 -07003749TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003750 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003751 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003752
3753 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003754 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003755 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003756 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3757 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003758
sprang57c2fff2017-01-16 06:24:02 -08003759 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003760 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003762 DataRate::BitsPerSec(kLowTargetBitrateBps),
3763 DataRate::BitsPerSec(kLowTargetBitrateBps),
3764 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003765
sprang57c2fff2017-01-16 06:24:02 -08003766 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003767 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3768 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003769 VideoBitrateAllocation bitrate_allocation =
3770 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003771 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003772 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003773 // TODO(srte): The use of millisecs here looks like an error, but the tests
3774 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003775 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003776
3777 // Not called on second frame.
3778 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3779 .Times(0);
3780 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003781 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3782 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003783 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003784
3785 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003786 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3787 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003788 const int64_t start_time_ms = rtc::TimeMillis();
3789 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3790 video_source_.IncomingCapturedFrame(
3791 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3792 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003793 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003794 }
3795
3796 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003797 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003798
mflodmancc3d4422017-08-03 08:27:51 -07003799 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003800}
3801
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003802TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3803 // 2 TLs configured, temporal layers supported by encoder.
3804 const int kNumTemporalLayers = 2;
3805 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3806 fake_encoder_.SetTemporalLayersSupported(0, true);
3807
3808 // Bitrate allocated across temporal layers.
3809 const int kTl0Bps = kTargetBitrateBps *
3810 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003811 kNumTemporalLayers, /*temporal_id*/ 0,
3812 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003813 const int kTl1Bps = kTargetBitrateBps *
3814 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003815 kNumTemporalLayers, /*temporal_id*/ 1,
3816 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003817 VideoBitrateAllocation expected_bitrate;
3818 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3819 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3820
3821 VerifyAllocatedBitrate(expected_bitrate);
3822 video_stream_encoder_->Stop();
3823}
3824
3825TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3826 // 2 TLs configured, temporal layers not supported by encoder.
3827 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3828 fake_encoder_.SetTemporalLayersSupported(0, false);
3829
3830 // Temporal layers not supported by the encoder.
3831 // Total bitrate should be at ti:0.
3832 VideoBitrateAllocation expected_bitrate;
3833 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3834
3835 VerifyAllocatedBitrate(expected_bitrate);
3836 video_stream_encoder_->Stop();
3837}
3838
3839TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3840 // 2 TLs configured, temporal layers only supported for first stream.
3841 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3842 fake_encoder_.SetTemporalLayersSupported(0, true);
3843 fake_encoder_.SetTemporalLayersSupported(1, false);
3844
3845 const int kS0Bps = 150000;
3846 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003847 kS0Bps *
3848 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3849 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003850 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003851 kS0Bps *
3852 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3853 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003854 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3855 // Temporal layers not supported by si:1.
3856 VideoBitrateAllocation expected_bitrate;
3857 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3858 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3859 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3860
3861 VerifyAllocatedBitrate(expected_bitrate);
3862 video_stream_encoder_->Stop();
3863}
3864
Niels Möller7dc26b72017-12-06 10:27:48 +01003865TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3866 const int kFrameWidth = 1280;
3867 const int kFrameHeight = 720;
3868 const int kFramerate = 24;
3869
Henrik Boström381d1092020-05-12 18:49:07 +02003870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003871 DataRate::BitsPerSec(kTargetBitrateBps),
3872 DataRate::BitsPerSec(kTargetBitrateBps),
3873 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003874 test::FrameForwarder source;
3875 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003876 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003877
3878 // Insert a single frame, triggering initial configuration.
3879 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3880 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3881
3882 EXPECT_EQ(
3883 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3884 kDefaultFramerate);
3885
3886 // Trigger reconfigure encoder (without resetting the entire instance).
3887 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003888 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003889 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3890 video_encoder_config.number_of_streams = 1;
3891 video_encoder_config.video_stream_factory =
3892 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3893 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003894 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003895 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3896
3897 // Detector should be updated with fps limit from codec config.
3898 EXPECT_EQ(
3899 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3900 kFramerate);
3901
3902 // Trigger overuse, max framerate should be reduced.
3903 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3904 stats.input_frame_rate = kFramerate;
3905 stats_proxy_->SetMockStats(stats);
3906 video_stream_encoder_->TriggerCpuOveruse();
3907 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3908 int adapted_framerate =
3909 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3910 EXPECT_LT(adapted_framerate, kFramerate);
3911
3912 // Trigger underuse, max framerate should go back to codec configured fps.
3913 // Set extra low fps, to make sure it's actually reset, not just incremented.
3914 stats = stats_proxy_->GetStats();
3915 stats.input_frame_rate = adapted_framerate / 2;
3916 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003917 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003918 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3919 EXPECT_EQ(
3920 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3921 kFramerate);
3922
3923 video_stream_encoder_->Stop();
3924}
3925
3926TEST_F(VideoStreamEncoderTest,
3927 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3928 const int kFrameWidth = 1280;
3929 const int kFrameHeight = 720;
3930 const int kLowFramerate = 15;
3931 const int kHighFramerate = 25;
3932
Henrik Boström381d1092020-05-12 18:49:07 +02003933 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003934 DataRate::BitsPerSec(kTargetBitrateBps),
3935 DataRate::BitsPerSec(kTargetBitrateBps),
3936 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003937 test::FrameForwarder source;
3938 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003939 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003940
3941 // Trigger initial configuration.
3942 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003943 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003944 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3945 video_encoder_config.number_of_streams = 1;
3946 video_encoder_config.video_stream_factory =
3947 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3948 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3949 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003950 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003951 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3952
3953 EXPECT_EQ(
3954 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3955 kLowFramerate);
3956
3957 // Trigger overuse, max framerate should be reduced.
3958 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3959 stats.input_frame_rate = kLowFramerate;
3960 stats_proxy_->SetMockStats(stats);
3961 video_stream_encoder_->TriggerCpuOveruse();
3962 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3963 int adapted_framerate =
3964 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3965 EXPECT_LT(adapted_framerate, kLowFramerate);
3966
3967 // Reconfigure the encoder with a new (higher max framerate), max fps should
3968 // still respect the adaptation.
3969 video_encoder_config.video_stream_factory =
3970 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3971 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3972 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003973 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003974 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3975
3976 EXPECT_EQ(
3977 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3978 adapted_framerate);
3979
3980 // Trigger underuse, max framerate should go back to codec configured fps.
3981 stats = stats_proxy_->GetStats();
3982 stats.input_frame_rate = adapted_framerate;
3983 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003984 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003985 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3986 EXPECT_EQ(
3987 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3988 kHighFramerate);
3989
3990 video_stream_encoder_->Stop();
3991}
3992
mflodmancc3d4422017-08-03 08:27:51 -07003993TEST_F(VideoStreamEncoderTest,
3994 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003995 const int kFrameWidth = 1280;
3996 const int kFrameHeight = 720;
3997 const int kFramerate = 24;
3998
Henrik Boström381d1092020-05-12 18:49:07 +02003999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004000 DataRate::BitsPerSec(kTargetBitrateBps),
4001 DataRate::BitsPerSec(kTargetBitrateBps),
4002 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07004003 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004004 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004005 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07004006
4007 // Trigger initial configuration.
4008 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004009 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07004010 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4011 video_encoder_config.number_of_streams = 1;
4012 video_encoder_config.video_stream_factory =
4013 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
4014 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004015 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004016 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004018
Niels Möller7dc26b72017-12-06 10:27:48 +01004019 EXPECT_EQ(
4020 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4021 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004022
4023 // Trigger overuse, max framerate should be reduced.
4024 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4025 stats.input_frame_rate = kFramerate;
4026 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004027 video_stream_encoder_->TriggerCpuOveruse();
4028 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004029 int adapted_framerate =
4030 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004031 EXPECT_LT(adapted_framerate, kFramerate);
4032
4033 // Change degradation preference to not enable framerate scaling. Target
4034 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004035 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004036 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004037 EXPECT_EQ(
4038 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4039 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004040
mflodmancc3d4422017-08-03 08:27:51 -07004041 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004042}
4043
mflodmancc3d4422017-08-03 08:27:51 -07004044TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004045 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004047 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4048 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4049 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004050 const int kWidth = 640;
4051 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004052
asaperssonfab67072017-04-04 05:51:49 -07004053 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004054
4055 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004056 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004057
4058 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004059 EXPECT_TRUE_WAIT(
4060 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004061
sprangc5d62e22017-04-02 23:53:04 -07004062 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004063
asaperssonfab67072017-04-04 05:51:49 -07004064 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004065 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004066 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004067
4068 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004069 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004070
Henrik Boström2671dac2020-05-19 16:29:09 +02004071 EXPECT_TRUE_WAIT(
4072 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004073
mflodmancc3d4422017-08-03 08:27:51 -07004074 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004075}
4076
mflodmancc3d4422017-08-03 08:27:51 -07004077TEST_F(VideoStreamEncoderTest,
4078 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004079 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004081 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4082 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4083 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004084 const int kWidth = 640;
4085 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004086
4087 // We expect the n initial frames to get dropped.
4088 int i;
4089 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004090 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004091 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004092 }
4093 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004094 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004095 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004096
4097 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004098 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004099
mflodmancc3d4422017-08-03 08:27:51 -07004100 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004101}
4102
mflodmancc3d4422017-08-03 08:27:51 -07004103TEST_F(VideoStreamEncoderTest,
4104 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004105 const int kWidth = 640;
4106 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004108 DataRate::BitsPerSec(kLowTargetBitrateBps),
4109 DataRate::BitsPerSec(kLowTargetBitrateBps),
4110 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004111
4112 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004113 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004114 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004115
asaperssonfab67072017-04-04 05:51:49 -07004116 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004117 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004118 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004119
mflodmancc3d4422017-08-03 08:27:51 -07004120 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004121}
4122
mflodmancc3d4422017-08-03 08:27:51 -07004123TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004124 const int kWidth = 640;
4125 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004126 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004127
4128 VideoEncoderConfig video_encoder_config;
4129 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4130 // Make format different, to force recreation of encoder.
4131 video_encoder_config.video_format.parameters["foo"] = "foo";
4132 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004133 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004135 DataRate::BitsPerSec(kLowTargetBitrateBps),
4136 DataRate::BitsPerSec(kLowTargetBitrateBps),
4137 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004138
kthelgasonb83797b2017-02-14 11:57:25 -08004139 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004140 video_stream_encoder_->SetSource(&video_source_,
4141 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004142
asaperssonfab67072017-04-04 05:51:49 -07004143 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004144 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004145 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004146
mflodmancc3d4422017-08-03 08:27:51 -07004147 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004148 fake_encoder_.SetQualityScaling(true);
4149}
4150
Åsa Persson139f4dc2019-08-02 09:29:58 +02004151TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4152 webrtc::test::ScopedFieldTrials field_trials(
4153 "WebRTC-Video-QualityScalerSettings/"
4154 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4155 // Reset encoder for field trials to take effect.
4156 ConfigureEncoder(video_encoder_config_.Copy());
4157 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4158 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4159 const int kWidth = 640;
4160 const int kHeight = 360;
4161
Henrik Boström381d1092020-05-12 18:49:07 +02004162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004163 DataRate::BitsPerSec(kTargetBitrateBps),
4164 DataRate::BitsPerSec(kTargetBitrateBps),
4165 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004166 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4167 // Frame should not be dropped.
4168 WaitForEncodedFrame(1);
4169
Henrik Boström381d1092020-05-12 18:49:07 +02004170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004171 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4172 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4173 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004174 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4175 // Frame should not be dropped.
4176 WaitForEncodedFrame(2);
4177
Henrik Boström381d1092020-05-12 18:49:07 +02004178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004179 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4180 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4181 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004182 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4183 // Expect to drop this frame, the wait should time out.
4184 ExpectDroppedFrame();
4185
4186 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004187 EXPECT_TRUE_WAIT(
4188 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004189 video_stream_encoder_->Stop();
4190}
4191
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004192TEST_F(VideoStreamEncoderTest,
4193 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
4194 webrtc::test::ScopedFieldTrials field_trials(
4195 "WebRTC-Video-QualityScalerSettings/"
4196 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4197 fake_encoder_.SetQualityScaling(false);
4198 ConfigureEncoder(video_encoder_config_.Copy());
4199 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4200 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4201 const int kWidth = 640;
4202 const int kHeight = 360;
4203
4204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4205 DataRate::BitsPerSec(kTargetBitrateBps),
4206 DataRate::BitsPerSec(kTargetBitrateBps),
4207 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4208 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4209 // Frame should not be dropped.
4210 WaitForEncodedFrame(1);
4211
4212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4213 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4214 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4215 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
4216 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4217 // Frame should not be dropped.
4218 WaitForEncodedFrame(2);
4219
4220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4221 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4222 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4223 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
4224 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4225 // Not dropped since quality scaling is disabled.
4226 WaitForEncodedFrame(3);
4227
4228 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02004229 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004230 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
4231
4232 video_stream_encoder_->Stop();
4233}
4234
Åsa Perssone644a032019-11-08 15:56:00 +01004235TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4236 webrtc::test::ScopedFieldTrials field_trials(
4237 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4238
4239 // Reset encoder for field trials to take effect.
4240 VideoEncoderConfig config = video_encoder_config_.Copy();
4241 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004242 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004243 ConfigureEncoder(std::move(config));
4244 fake_encoder_.SetQp(kQpLow);
4245
4246 // Enable MAINTAIN_FRAMERATE preference.
4247 AdaptingFrameForwarder source;
4248 source.set_adaptation_enabled(true);
4249 video_stream_encoder_->SetSource(&source,
4250 DegradationPreference::MAINTAIN_FRAMERATE);
4251
4252 // Start at low bitrate.
4253 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4255 DataRate::BitsPerSec(kLowBitrateBps),
4256 DataRate::BitsPerSec(kLowBitrateBps),
4257 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004258
4259 // Expect first frame to be dropped and resolution to be limited.
4260 const int kWidth = 1280;
4261 const int kHeight = 720;
4262 const int64_t kFrameIntervalMs = 100;
4263 int64_t timestamp_ms = kFrameIntervalMs;
4264 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4265 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004266 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4267 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004268
4269 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004270 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4271 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004272
4273 // Insert frames and advance |min_duration_ms|.
4274 for (size_t i = 1; i <= 10; i++) {
4275 timestamp_ms += kFrameIntervalMs;
4276 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4277 WaitForEncodedFrame(timestamp_ms);
4278 }
4279 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4280 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4281
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004282 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004283
4284 // Insert frame should trigger high BW and release quality limitation.
4285 timestamp_ms += kFrameIntervalMs;
4286 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4287 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004288 // The ramp-up code involves the adaptation queue, give it time to execute.
4289 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02004290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004291 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004292
4293 // Frame should not be adapted.
4294 timestamp_ms += kFrameIntervalMs;
4295 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4296 WaitForEncodedFrame(kWidth, kHeight);
4297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4298
4299 video_stream_encoder_->Stop();
4300}
4301
mflodmancc3d4422017-08-03 08:27:51 -07004302TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004303 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
4304 AdaptingFrameForwarder source;
4305 source.set_adaptation_enabled(true);
4306 video_stream_encoder_->SetSource(&source,
4307 DegradationPreference::MAINTAIN_FRAMERATE);
4308 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4309 DataRate::BitsPerSec(kTargetBitrateBps),
4310 DataRate::BitsPerSec(kTargetBitrateBps),
4311 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4312 fake_encoder_.SetQp(kQpHigh + 1);
4313 const int kWidth = 1280;
4314 const int kHeight = 720;
4315 const int64_t kFrameIntervalMs = 100;
4316 int64_t timestamp_ms = kFrameIntervalMs;
4317 for (size_t i = 1; i <= 100; i++) {
4318 timestamp_ms += kFrameIntervalMs;
4319 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4320 WaitForEncodedFrame(timestamp_ms);
4321 }
4322 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
4323 // for the first time.
4324 // TODO(eshr): We should avoid these waits by using threads with simulated
4325 // time.
4326 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
4327 2000 * 2.5 * 2);
4328 timestamp_ms += kFrameIntervalMs;
4329 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4330 WaitForEncodedFrame(timestamp_ms);
4331 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4332 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
4333 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4334
4335 // Disable Quality scaling by turning off scaler on the encoder and
4336 // reconfiguring.
4337 fake_encoder_.SetQualityScaling(false);
4338 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
4339 kMaxPayloadLength);
4340 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4341 // Since we turned off the quality scaler, the adaptations made by it are
4342 // removed.
4343 EXPECT_THAT(source.sink_wants(), ResolutionMax());
4344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4345
4346 video_stream_encoder_->Stop();
4347}
4348
4349TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004350 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4351 const int kTooSmallWidth = 10;
4352 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004354 DataRate::BitsPerSec(kTargetBitrateBps),
4355 DataRate::BitsPerSec(kTargetBitrateBps),
4356 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004357
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004358 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004359 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004360 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004361 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004362 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004363 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4364
4365 // Trigger adapt down, too small frame, expect no change.
4366 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004367 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004368 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004369 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004370 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4371 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4372
mflodmancc3d4422017-08-03 08:27:51 -07004373 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004374}
4375
mflodmancc3d4422017-08-03 08:27:51 -07004376TEST_F(VideoStreamEncoderTest,
4377 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004378 const int kTooSmallWidth = 10;
4379 const int kTooSmallHeight = 10;
4380 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004381 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004382 DataRate::BitsPerSec(kTargetBitrateBps),
4383 DataRate::BitsPerSec(kTargetBitrateBps),
4384 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004385
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004386 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004387 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004388 video_stream_encoder_->SetSource(&source,
4389 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004390 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004391 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4392 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4393
4394 // Trigger adapt down, expect limited framerate.
4395 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004396 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004397 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004398 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004399 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4401 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4402
4403 // Trigger adapt down, too small frame, expect no change.
4404 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004405 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004406 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004407 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004408 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4410 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4411
mflodmancc3d4422017-08-03 08:27:51 -07004412 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004413}
4414
mflodmancc3d4422017-08-03 08:27:51 -07004415TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004416 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004418 DataRate::BitsPerSec(kTargetBitrateBps),
4419 DataRate::BitsPerSec(kTargetBitrateBps),
4420 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004421 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004422 const int kFrameWidth = 1280;
4423 const int kFrameHeight = 720;
4424 video_source_.IncomingCapturedFrame(
4425 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004426 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004427 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004428}
4429
sprangb1ca0732017-02-01 08:38:12 -08004430// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004431TEST_F(VideoStreamEncoderTest,
4432 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004434 DataRate::BitsPerSec(kTargetBitrateBps),
4435 DataRate::BitsPerSec(kTargetBitrateBps),
4436 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004437
4438 const int kFrameWidth = 1280;
4439 const int kFrameHeight = 720;
4440 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004441 // requested by
4442 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004443 video_source_.set_adaptation_enabled(true);
4444
4445 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004446 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004447 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004448
4449 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004450 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004451 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004452 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004453 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004454
asaperssonfab67072017-04-04 05:51:49 -07004455 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004456 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004457 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004458 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004459 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004460
mflodmancc3d4422017-08-03 08:27:51 -07004461 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004462}
sprangfe627f32017-03-29 08:24:59 -07004463
mflodmancc3d4422017-08-03 08:27:51 -07004464TEST_F(VideoStreamEncoderTest,
4465 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004466 const int kFrameWidth = 1280;
4467 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004468
Henrik Boström381d1092020-05-12 18:49:07 +02004469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004470 DataRate::BitsPerSec(kTargetBitrateBps),
4471 DataRate::BitsPerSec(kTargetBitrateBps),
4472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004473 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004474 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004475 video_source_.set_adaptation_enabled(true);
4476
sprang4847ae62017-06-27 07:06:52 -07004477 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004478
4479 video_source_.IncomingCapturedFrame(
4480 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004481 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004482
4483 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004484 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004485
4486 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004487 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004488 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004489 video_source_.IncomingCapturedFrame(
4490 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004491 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004492 }
4493
4494 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004495 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004496 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004497 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004498 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004499 video_source_.IncomingCapturedFrame(
4500 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004501 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004502 ++num_frames_dropped;
4503 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004504 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004505 }
4506 }
4507
sprang4847ae62017-06-27 07:06:52 -07004508 // Add some slack to account for frames dropped by the frame dropper.
4509 const int kErrorMargin = 1;
4510 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004511 kErrorMargin);
4512
4513 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004514 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004515 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004516 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004517 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004518 video_source_.IncomingCapturedFrame(
4519 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004520 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004521 ++num_frames_dropped;
4522 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004523 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004524 }
4525 }
sprang4847ae62017-06-27 07:06:52 -07004526 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004527 kErrorMargin);
4528
4529 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004530 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004531 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004532 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004533 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004534 video_source_.IncomingCapturedFrame(
4535 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004536 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004537 ++num_frames_dropped;
4538 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004539 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004540 }
4541 }
sprang4847ae62017-06-27 07:06:52 -07004542 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004543 kErrorMargin);
4544
4545 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004546 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004547 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004548 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004549 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004550 video_source_.IncomingCapturedFrame(
4551 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004552 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004553 ++num_frames_dropped;
4554 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004555 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004556 }
4557 }
4558 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4559
mflodmancc3d4422017-08-03 08:27:51 -07004560 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004561}
4562
mflodmancc3d4422017-08-03 08:27:51 -07004563TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004564 const int kFramerateFps = 5;
4565 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004566 const int kFrameWidth = 1280;
4567 const int kFrameHeight = 720;
4568
sprang4847ae62017-06-27 07:06:52 -07004569 // Reconfigure encoder with two temporal layers and screensharing, which will
4570 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004571 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004572
Henrik Boström381d1092020-05-12 18:49:07 +02004573 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004574 DataRate::BitsPerSec(kTargetBitrateBps),
4575 DataRate::BitsPerSec(kTargetBitrateBps),
4576 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004577 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004578 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004579 video_source_.set_adaptation_enabled(true);
4580
sprang4847ae62017-06-27 07:06:52 -07004581 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004582
4583 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004584 rtc::VideoSinkWants last_wants;
4585 do {
4586 last_wants = video_source_.sink_wants();
4587
sprangc5d62e22017-04-02 23:53:04 -07004588 // Insert frames to get a new fps estimate...
4589 for (int j = 0; j < kFramerateFps; ++j) {
4590 video_source_.IncomingCapturedFrame(
4591 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004592 if (video_source_.last_sent_width()) {
4593 sink_.WaitForEncodedFrame(timestamp_ms);
4594 }
sprangc5d62e22017-04-02 23:53:04 -07004595 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004596 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004597 }
4598 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004599 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004600 } while (video_source_.sink_wants().max_framerate_fps <
4601 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004602
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004603 EXPECT_THAT(video_source_.sink_wants(),
4604 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004605
mflodmancc3d4422017-08-03 08:27:51 -07004606 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004607}
asaperssonf7e294d2017-06-13 23:25:22 -07004608
mflodmancc3d4422017-08-03 08:27:51 -07004609TEST_F(VideoStreamEncoderTest,
4610 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004611 const int kWidth = 1280;
4612 const int kHeight = 720;
4613 const int64_t kFrameIntervalMs = 150;
4614 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004616 DataRate::BitsPerSec(kTargetBitrateBps),
4617 DataRate::BitsPerSec(kTargetBitrateBps),
4618 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004619
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004620 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004621 AdaptingFrameForwarder source;
4622 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004623 video_stream_encoder_->SetSource(&source,
4624 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004625 timestamp_ms += kFrameIntervalMs;
4626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004627 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004628 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004629 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4630 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4631 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4632
4633 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004634 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004635 timestamp_ms += kFrameIntervalMs;
4636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004637 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004638 EXPECT_THAT(source.sink_wants(),
4639 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4642 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4643
4644 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004645 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004646 timestamp_ms += kFrameIntervalMs;
4647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004648 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004649 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004650 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4652 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4653
4654 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004655 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004656 timestamp_ms += kFrameIntervalMs;
4657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004658 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004659 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4662 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4663
4664 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004665 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004666 timestamp_ms += kFrameIntervalMs;
4667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004668 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004669 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4671 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4672 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4673
4674 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004675 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004676 timestamp_ms += kFrameIntervalMs;
4677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004678 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004679 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004680 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4681 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4682 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4683
4684 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004685 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004686 timestamp_ms += kFrameIntervalMs;
4687 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004688 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004689 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004690 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4691 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4692 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4693
4694 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004695 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004696 timestamp_ms += kFrameIntervalMs;
4697 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004698 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004699 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004700 rtc::VideoSinkWants last_wants = source.sink_wants();
4701 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4702 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4703 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4704
4705 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004706 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004707 timestamp_ms += kFrameIntervalMs;
4708 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004709 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004710 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004711 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4712 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4713 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4714
Evan Shrubsole64469032020-06-11 10:45:29 +02004715 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004716 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004717 timestamp_ms += kFrameIntervalMs;
4718 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004719 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004720 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004721 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4722 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4723 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4724
4725 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004726 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004727 timestamp_ms += kFrameIntervalMs;
4728 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004729 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004730 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004731 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4732 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4733 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4734
4735 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004736 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004737 timestamp_ms += kFrameIntervalMs;
4738 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004739 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004740 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004741 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4742 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4743 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4744
4745 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004746 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004747 timestamp_ms += kFrameIntervalMs;
4748 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004749 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004750 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004751 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4752 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4753 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4754
4755 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004756 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004757 timestamp_ms += kFrameIntervalMs;
4758 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004759 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004760 EXPECT_THAT(source.sink_wants(), FpsMax());
4761 EXPECT_EQ(source.sink_wants().max_pixel_count,
4762 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004763 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4764 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4765 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4766
4767 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004768 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004769 timestamp_ms += kFrameIntervalMs;
4770 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004771 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004772 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004773 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4774 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4775 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4776
Åsa Persson30ab0152019-08-27 12:22:33 +02004777 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004778 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004779 timestamp_ms += kFrameIntervalMs;
4780 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004781 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004782 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004783 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004784 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4785 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4786 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4787
4788 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004789 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004790 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004791 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4792
mflodmancc3d4422017-08-03 08:27:51 -07004793 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004794}
4795
mflodmancc3d4422017-08-03 08:27:51 -07004796TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004797 const int kWidth = 1280;
4798 const int kHeight = 720;
4799 const int64_t kFrameIntervalMs = 150;
4800 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004802 DataRate::BitsPerSec(kTargetBitrateBps),
4803 DataRate::BitsPerSec(kTargetBitrateBps),
4804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004805
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004806 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004807 AdaptingFrameForwarder source;
4808 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004809 video_stream_encoder_->SetSource(&source,
4810 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004811 timestamp_ms += kFrameIntervalMs;
4812 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004813 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004815 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4817 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4818 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4819 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4820 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4821
4822 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004823 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004824 timestamp_ms += kFrameIntervalMs;
4825 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004826 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004827 EXPECT_THAT(source.sink_wants(),
4828 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004829 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4830 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4831 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4832 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4833 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4834 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4835
4836 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004837 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004838 timestamp_ms += kFrameIntervalMs;
4839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004840 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004841 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4844 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4846 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4847 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4848
4849 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004850 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004851 timestamp_ms += kFrameIntervalMs;
4852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004853 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004854 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02004855 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07004856 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4857 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4858 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4859 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4861
Evan Shrubsole64469032020-06-11 10:45:29 +02004862 // Trigger cpu adapt up, expect no change since QP is most limited.
4863 {
4864 // Store current sink wants since we expect no change and if there is no
4865 // change then last_wants() is not updated.
4866 auto previous_sink_wants = source.sink_wants();
4867 video_stream_encoder_->TriggerCpuUnderuse();
4868 timestamp_ms += kFrameIntervalMs;
4869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4870 WaitForEncodedFrame(timestamp_ms);
4871 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4872 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4874 }
4875
4876 // Trigger quality adapt up, expect increased fps (640x360@30fps).
4877 video_stream_encoder_->TriggerQualityHigh();
4878 timestamp_ms += kFrameIntervalMs;
4879 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4880 WaitForEncodedFrame(timestamp_ms);
4881 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
4882 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4884 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4885 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4886 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4887 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4888
4889 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4890 // expect increased resolution (960x540@30fps).
4891 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004892 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004893 timestamp_ms += kFrameIntervalMs;
4894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004895 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02004896 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004897 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4899 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4900 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4901 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004902 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004903
Evan Shrubsole64469032020-06-11 10:45:29 +02004904 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4905 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004906 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004907 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004908 timestamp_ms += kFrameIntervalMs;
4909 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004910 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004911 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004912 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004913 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4914 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4915 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4916 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4917 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004918 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004919
4920 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004921 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004922 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004923 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004924 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004925
mflodmancc3d4422017-08-03 08:27:51 -07004926 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004927}
4928
mflodmancc3d4422017-08-03 08:27:51 -07004929TEST_F(VideoStreamEncoderTest,
4930 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004931 const int kWidth = 640;
4932 const int kHeight = 360;
4933 const int kFpsLimit = 15;
4934 const int64_t kFrameIntervalMs = 150;
4935 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004936 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004937 DataRate::BitsPerSec(kTargetBitrateBps),
4938 DataRate::BitsPerSec(kTargetBitrateBps),
4939 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004940
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004941 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004942 AdaptingFrameForwarder source;
4943 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004944 video_stream_encoder_->SetSource(&source,
4945 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004946 timestamp_ms += kFrameIntervalMs;
4947 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004948 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004949 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004950 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4951 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4952 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4953 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4954 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4955 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4956
4957 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004958 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004959 timestamp_ms += kFrameIntervalMs;
4960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004961 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004962 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004963 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4964 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4965 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4966 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4967 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4968 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4969
4970 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004971 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004972 timestamp_ms += kFrameIntervalMs;
4973 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004974 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004975 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004976 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004977 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07004978 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4979 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4980 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4981 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4982
Evan Shrubsole64469032020-06-11 10:45:29 +02004983 // Trigger cpu adapt up, expect no change because quality is most limited.
4984 {
4985 auto previous_sink_wants = source.sink_wants();
4986 // Store current sink wants since we expect no change ind if there is no
4987 // change then last__wants() is not updated.
4988 video_stream_encoder_->TriggerCpuUnderuse();
4989 timestamp_ms += kFrameIntervalMs;
4990 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4991 WaitForEncodedFrame(timestamp_ms);
4992 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4993 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4994 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4995 }
4996
4997 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
4998 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004999 timestamp_ms += kFrameIntervalMs;
5000 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005001 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005002 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005003 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5004 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5005 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02005006 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5007 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5008 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005009
Evan Shrubsole64469032020-06-11 10:45:29 +02005010 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005011 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02005012 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005013 timestamp_ms += kFrameIntervalMs;
5014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005015 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005016 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5018 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5019 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5020 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5021 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005022 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005023
5024 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005025 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005026 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005027 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005028 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005029
mflodmancc3d4422017-08-03 08:27:51 -07005030 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005031}
5032
mflodmancc3d4422017-08-03 08:27:51 -07005033TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07005034 const int kFrameWidth = 1920;
5035 const int kFrameHeight = 1080;
5036 // 3/4 of 1920.
5037 const int kAdaptedFrameWidth = 1440;
5038 // 3/4 of 1080 rounded down to multiple of 4.
5039 const int kAdaptedFrameHeight = 808;
5040 const int kFramerate = 24;
5041
Henrik Boström381d1092020-05-12 18:49:07 +02005042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005043 DataRate::BitsPerSec(kTargetBitrateBps),
5044 DataRate::BitsPerSec(kTargetBitrateBps),
5045 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07005046 // Trigger reconfigure encoder (without resetting the entire instance).
5047 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02005048 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07005049 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
5050 video_encoder_config.number_of_streams = 1;
5051 video_encoder_config.video_stream_factory =
5052 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07005053 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005054 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005055 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07005056
5057 video_source_.set_adaptation_enabled(true);
5058
5059 video_source_.IncomingCapturedFrame(
5060 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005061 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005062
5063 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005064 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07005065 video_source_.IncomingCapturedFrame(
5066 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005067 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005068
mflodmancc3d4422017-08-03 08:27:51 -07005069 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07005070}
5071
mflodmancc3d4422017-08-03 08:27:51 -07005072TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07005073 const int kFrameWidth = 1280;
5074 const int kFrameHeight = 720;
5075 const int kLowFps = 2;
5076 const int kHighFps = 30;
5077
Henrik Boström381d1092020-05-12 18:49:07 +02005078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005079 DataRate::BitsPerSec(kTargetBitrateBps),
5080 DataRate::BitsPerSec(kTargetBitrateBps),
5081 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005082
5083 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5084 max_framerate_ = kLowFps;
5085
5086 // Insert 2 seconds of 2fps video.
5087 for (int i = 0; i < kLowFps * 2; ++i) {
5088 video_source_.IncomingCapturedFrame(
5089 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5090 WaitForEncodedFrame(timestamp_ms);
5091 timestamp_ms += 1000 / kLowFps;
5092 }
5093
5094 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02005095 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005096 DataRate::BitsPerSec(kTargetBitrateBps),
5097 DataRate::BitsPerSec(kTargetBitrateBps),
5098 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005099 video_source_.IncomingCapturedFrame(
5100 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5101 WaitForEncodedFrame(timestamp_ms);
5102 timestamp_ms += 1000 / kLowFps;
5103
5104 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
5105
5106 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02005107 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07005108 const int kFrameIntervalMs = 1000 / kHighFps;
5109 max_framerate_ = kHighFps;
5110 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
5111 video_source_.IncomingCapturedFrame(
5112 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5113 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
5114 // be dropped if the encoder hans't been updated with the new higher target
5115 // framerate yet, causing it to overshoot the target bitrate and then
5116 // suffering the wrath of the media optimizer.
5117 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
5118 timestamp_ms += kFrameIntervalMs;
5119 }
5120
5121 // Don expect correct measurement just yet, but it should be higher than
5122 // before.
5123 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
5124
mflodmancc3d4422017-08-03 08:27:51 -07005125 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005126}
5127
mflodmancc3d4422017-08-03 08:27:51 -07005128TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07005129 const int kFrameWidth = 1280;
5130 const int kFrameHeight = 720;
5131 const int kTargetBitrateBps = 1000000;
5132
5133 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02005134 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02005135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005136 DataRate::BitsPerSec(kTargetBitrateBps),
5137 DataRate::BitsPerSec(kTargetBitrateBps),
5138 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005139 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005140
5141 // Insert a first video frame, causes another bitrate update.
5142 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5143 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5144 video_source_.IncomingCapturedFrame(
5145 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5146 WaitForEncodedFrame(timestamp_ms);
5147
5148 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005149 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5150 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5151 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005152
5153 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005154 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005155 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005156
5157 // Bitrate observer should not be called.
5158 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5159 video_source_.IncomingCapturedFrame(
5160 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5161 ExpectDroppedFrame();
5162
mflodmancc3d4422017-08-03 08:27:51 -07005163 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005164}
ilnik6b826ef2017-06-16 06:53:48 -07005165
Niels Möller4db138e2018-04-19 09:04:13 +02005166TEST_F(VideoStreamEncoderTest,
5167 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5168 const int kFrameWidth = 1280;
5169 const int kFrameHeight = 720;
5170 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005171 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005172 DataRate::BitsPerSec(kTargetBitrateBps),
5173 DataRate::BitsPerSec(kTargetBitrateBps),
5174 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005175 video_source_.IncomingCapturedFrame(
5176 CreateFrame(1, kFrameWidth, kFrameHeight));
5177 WaitForEncodedFrame(1);
5178 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5179 .low_encode_usage_threshold_percent,
5180 default_options.low_encode_usage_threshold_percent);
5181 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5182 .high_encode_usage_threshold_percent,
5183 default_options.high_encode_usage_threshold_percent);
5184 video_stream_encoder_->Stop();
5185}
5186
5187TEST_F(VideoStreamEncoderTest,
5188 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5189 const int kFrameWidth = 1280;
5190 const int kFrameHeight = 720;
5191 CpuOveruseOptions hardware_options;
5192 hardware_options.low_encode_usage_threshold_percent = 150;
5193 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005194 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005195
Henrik Boström381d1092020-05-12 18:49:07 +02005196 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005197 DataRate::BitsPerSec(kTargetBitrateBps),
5198 DataRate::BitsPerSec(kTargetBitrateBps),
5199 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005200 video_source_.IncomingCapturedFrame(
5201 CreateFrame(1, kFrameWidth, kFrameHeight));
5202 WaitForEncodedFrame(1);
5203 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5204 .low_encode_usage_threshold_percent,
5205 hardware_options.low_encode_usage_threshold_percent);
5206 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5207 .high_encode_usage_threshold_percent,
5208 hardware_options.high_encode_usage_threshold_percent);
5209 video_stream_encoder_->Stop();
5210}
5211
Niels Möller6bb5ab92019-01-11 11:11:10 +01005212TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5213 const int kFrameWidth = 320;
5214 const int kFrameHeight = 240;
5215 const int kFps = 30;
5216 const int kTargetBitrateBps = 120000;
5217 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5218
Henrik Boström381d1092020-05-12 18:49:07 +02005219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005220 DataRate::BitsPerSec(kTargetBitrateBps),
5221 DataRate::BitsPerSec(kTargetBitrateBps),
5222 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005223
5224 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5225 max_framerate_ = kFps;
5226
5227 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5228 fake_encoder_.SimulateOvershoot(1.0);
5229 int num_dropped = 0;
5230 for (int i = 0; i < kNumFramesInRun; ++i) {
5231 video_source_.IncomingCapturedFrame(
5232 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5233 // Wait up to two frame durations for a frame to arrive.
5234 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5235 ++num_dropped;
5236 }
5237 timestamp_ms += 1000 / kFps;
5238 }
5239
Erik Språnga8d48ab2019-02-08 14:17:40 +01005240 // Framerate should be measured to be near the expected target rate.
5241 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5242
5243 // Frame drops should be within 5% of expected 0%.
5244 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005245
5246 // Make encoder produce frames at double the expected bitrate during 3 seconds
5247 // of video, verify number of drops. Rate needs to be slightly changed in
5248 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005249 double overshoot_factor = 2.0;
5250 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5251 // With bitrate adjuster, when need to overshoot even more to trigger
5252 // frame dropping.
5253 overshoot_factor *= 2;
5254 }
5255 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005257 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5258 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5259 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005260 num_dropped = 0;
5261 for (int i = 0; i < kNumFramesInRun; ++i) {
5262 video_source_.IncomingCapturedFrame(
5263 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5264 // Wait up to two frame durations for a frame to arrive.
5265 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5266 ++num_dropped;
5267 }
5268 timestamp_ms += 1000 / kFps;
5269 }
5270
Henrik Boström381d1092020-05-12 18:49:07 +02005271 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005272 DataRate::BitsPerSec(kTargetBitrateBps),
5273 DataRate::BitsPerSec(kTargetBitrateBps),
5274 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005275
5276 // Target framerate should be still be near the expected target, despite
5277 // the frame drops.
5278 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5279
5280 // Frame drops should be within 5% of expected 50%.
5281 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005282
5283 video_stream_encoder_->Stop();
5284}
5285
5286TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5287 const int kFrameWidth = 320;
5288 const int kFrameHeight = 240;
5289 const int kActualInputFps = 24;
5290 const int kTargetBitrateBps = 120000;
5291
5292 ASSERT_GT(max_framerate_, kActualInputFps);
5293
5294 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5295 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005297 DataRate::BitsPerSec(kTargetBitrateBps),
5298 DataRate::BitsPerSec(kTargetBitrateBps),
5299 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005300
5301 // Insert 3 seconds of video, with an input fps lower than configured max.
5302 for (int i = 0; i < kActualInputFps * 3; ++i) {
5303 video_source_.IncomingCapturedFrame(
5304 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5305 // Wait up to two frame durations for a frame to arrive.
5306 WaitForEncodedFrame(timestamp_ms);
5307 timestamp_ms += 1000 / kActualInputFps;
5308 }
5309
5310 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5311
5312 video_stream_encoder_->Stop();
5313}
5314
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005315TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5316 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005318 DataRate::BitsPerSec(kTargetBitrateBps),
5319 DataRate::BitsPerSec(kTargetBitrateBps),
5320 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005321
5322 fake_encoder_.BlockNextEncode();
5323 video_source_.IncomingCapturedFrame(
5324 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5325 WaitForEncodedFrame(1);
5326 // On the very first frame full update should be forced.
5327 rect = fake_encoder_.GetLastUpdateRect();
5328 EXPECT_EQ(rect.offset_x, 0);
5329 EXPECT_EQ(rect.offset_y, 0);
5330 EXPECT_EQ(rect.height, codec_height_);
5331 EXPECT_EQ(rect.width, codec_width_);
5332 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5333 // call to ContinueEncode.
5334 video_source_.IncomingCapturedFrame(
5335 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5336 ExpectDroppedFrame();
5337 video_source_.IncomingCapturedFrame(
5338 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5339 ExpectDroppedFrame();
5340 fake_encoder_.ContinueEncode();
5341 WaitForEncodedFrame(3);
5342 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5343 rect = fake_encoder_.GetLastUpdateRect();
5344 EXPECT_EQ(rect.offset_x, 1);
5345 EXPECT_EQ(rect.offset_y, 0);
5346 EXPECT_EQ(rect.width, 10);
5347 EXPECT_EQ(rect.height, 1);
5348
5349 video_source_.IncomingCapturedFrame(
5350 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5351 WaitForEncodedFrame(4);
5352 // Previous frame was encoded, so no accumulation should happen.
5353 rect = fake_encoder_.GetLastUpdateRect();
5354 EXPECT_EQ(rect.offset_x, 0);
5355 EXPECT_EQ(rect.offset_y, 0);
5356 EXPECT_EQ(rect.width, 1);
5357 EXPECT_EQ(rect.height, 1);
5358
5359 video_stream_encoder_->Stop();
5360}
5361
Erik Språngd7329ca2019-02-21 21:19:53 +01005362TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005363 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005364 DataRate::BitsPerSec(kTargetBitrateBps),
5365 DataRate::BitsPerSec(kTargetBitrateBps),
5366 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005367
5368 // First frame is always keyframe.
5369 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5370 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005371 EXPECT_THAT(
5372 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005373 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005374
5375 // Insert delta frame.
5376 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5377 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005378 EXPECT_THAT(
5379 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005380 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005381
5382 // Request next frame be a key-frame.
5383 video_stream_encoder_->SendKeyFrame();
5384 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5385 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005386 EXPECT_THAT(
5387 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005388 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005389
5390 video_stream_encoder_->Stop();
5391}
5392
5393TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5394 // Setup simulcast with three streams.
5395 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005397 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5398 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5399 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005400 // Wait for all three layers before triggering event.
5401 sink_.SetNumExpectedLayers(3);
5402
5403 // First frame is always keyframe.
5404 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5405 WaitForEncodedFrame(1);
5406 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005407 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5408 VideoFrameType::kVideoFrameKey,
5409 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005410
5411 // Insert delta frame.
5412 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5413 WaitForEncodedFrame(2);
5414 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005415 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5416 VideoFrameType::kVideoFrameDelta,
5417 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005418
5419 // Request next frame be a key-frame.
5420 // Only first stream is configured to produce key-frame.
5421 video_stream_encoder_->SendKeyFrame();
5422 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5423 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005424
5425 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5426 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005427 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005428 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005429 VideoFrameType::kVideoFrameKey,
5430 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005431
5432 video_stream_encoder_->Stop();
5433}
5434
5435TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5436 // Configure internal source factory and setup test again.
5437 encoder_factory_.SetHasInternalSource(true);
5438 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005440 DataRate::BitsPerSec(kTargetBitrateBps),
5441 DataRate::BitsPerSec(kTargetBitrateBps),
5442 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005443
5444 // Call encoder directly, simulating internal source where encoded frame
5445 // callback in VideoStreamEncoder is called despite no OnFrame().
5446 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5447 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005448 EXPECT_THAT(
5449 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005450 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005451
Niels Möller8f7ce222019-03-21 15:43:58 +01005452 const std::vector<VideoFrameType> kDeltaFrame = {
5453 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005454 // Need to set timestamp manually since manually for injected frame.
5455 VideoFrame frame = CreateFrame(101, nullptr);
5456 frame.set_timestamp(101);
5457 fake_encoder_.InjectFrame(frame, false);
5458 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005459 EXPECT_THAT(
5460 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005461 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005462
5463 // Request key-frame. The forces a dummy frame down into the encoder.
5464 fake_encoder_.ExpectNullFrame();
5465 video_stream_encoder_->SendKeyFrame();
5466 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005467 EXPECT_THAT(
5468 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005469 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005470
5471 video_stream_encoder_->Stop();
5472}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005473
5474TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5475 // Configure internal source factory and setup test again.
5476 encoder_factory_.SetHasInternalSource(true);
5477 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005479 DataRate::BitsPerSec(kTargetBitrateBps),
5480 DataRate::BitsPerSec(kTargetBitrateBps),
5481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005482
5483 int64_t timestamp = 1;
5484 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005485 image.SetEncodedData(
5486 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005487 image.capture_time_ms_ = ++timestamp;
5488 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5489 const int64_t kEncodeFinishDelayMs = 10;
5490 image.timing_.encode_start_ms = timestamp;
5491 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5492 fake_encoder_.InjectEncodedImage(image);
5493 // Wait for frame without incrementing clock.
5494 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5495 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5496 // capture timestamp should be kEncodeFinishDelayMs in the past.
5497 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5498 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5499 kEncodeFinishDelayMs);
5500
5501 video_stream_encoder_->Stop();
5502}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005503
5504TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005505 // SPS contains VUI with restrictions on the maximum number of reordered
5506 // pictures, there is no need to rewrite the bitstream to enable faster
5507 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005508 ResetEncoder("H264", 1, 1, 1, false);
5509
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5511 DataRate::BitsPerSec(kTargetBitrateBps),
5512 DataRate::BitsPerSec(kTargetBitrateBps),
5513 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5514 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005515
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005516 fake_encoder_.SetEncodedImageData(
5517 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005518
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005519 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5520 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005521
5522 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5523 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005524
5525 video_stream_encoder_->Stop();
5526}
5527
5528TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005529 // SPS does not contain VUI, the bitstream is will be rewritten with added
5530 // VUI with restrictions on the maximum number of reordered pictures to
5531 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005532 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5533 0x00, 0x00, 0x03, 0x03, 0xF4,
5534 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005535 ResetEncoder("H264", 1, 1, 1, false);
5536
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005537 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5538 DataRate::BitsPerSec(kTargetBitrateBps),
5539 DataRate::BitsPerSec(kTargetBitrateBps),
5540 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5541 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005542
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005543 fake_encoder_.SetEncodedImageData(
5544 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005545
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005546 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5547 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005548
5549 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5550 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005551
5552 video_stream_encoder_->Stop();
5553}
5554
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005555TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5556 const int kFrameWidth = 1280;
5557 const int kFrameHeight = 720;
5558 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5559
Henrik Boström381d1092020-05-12 18:49:07 +02005560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005561 DataRate::BitsPerSec(kTargetBitrateBps),
5562 DataRate::BitsPerSec(kTargetBitrateBps),
5563 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005564 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5565
5566 // Insert a first video frame. It should be dropped because of downscale in
5567 // resolution.
5568 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5569 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5570 frame.set_rotation(kVideoRotation_270);
5571 video_source_.IncomingCapturedFrame(frame);
5572
5573 ExpectDroppedFrame();
5574
5575 // Second frame is downscaled.
5576 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5577 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5578 frame.set_rotation(kVideoRotation_90);
5579 video_source_.IncomingCapturedFrame(frame);
5580
5581 WaitForEncodedFrame(timestamp_ms);
5582 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5583
5584 // Insert another frame, also downscaled.
5585 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5586 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5587 frame.set_rotation(kVideoRotation_180);
5588 video_source_.IncomingCapturedFrame(frame);
5589
5590 WaitForEncodedFrame(timestamp_ms);
5591 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5592
5593 video_stream_encoder_->Stop();
5594}
5595
Erik Språng5056af02019-09-02 15:53:11 +02005596TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5597 const int kFrameWidth = 320;
5598 const int kFrameHeight = 180;
5599
5600 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005602 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5603 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5604 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005605 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005606 /*rtt_ms=*/0,
5607 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005608
5609 // Insert a first video frame so that encoder gets configured.
5610 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5611 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5612 frame.set_rotation(kVideoRotation_270);
5613 video_source_.IncomingCapturedFrame(frame);
5614 WaitForEncodedFrame(timestamp_ms);
5615
5616 // Set a target rate below the minimum allowed by the codec settings.
5617 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005618 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5619 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005621 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005622 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005623 /*link_allocation=*/target_rate,
5624 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005625 /*rtt_ms=*/0,
5626 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005627 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5628
5629 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5630 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5631 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005632 DataRate allocation_sum =
5633 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005634 EXPECT_EQ(min_rate, allocation_sum);
5635 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5636
5637 video_stream_encoder_->Stop();
5638}
5639
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005640TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005642 DataRate::BitsPerSec(kTargetBitrateBps),
5643 DataRate::BitsPerSec(kTargetBitrateBps),
5644 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005645 // Capture a frame and wait for it to synchronize with the encoder thread.
5646 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5647 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5648 WaitForEncodedFrame(1);
5649
5650 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5651 ASSERT_TRUE(prev_rate_settings.has_value());
5652 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5653 kDefaultFramerate);
5654
5655 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5656 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5657 timestamp_ms += 1000 / kDefaultFramerate;
5658 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5659 WaitForEncodedFrame(timestamp_ms);
5660 }
5661 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5662 kDefaultFramerate);
5663 // Capture larger frame to trigger a reconfigure.
5664 codec_height_ *= 2;
5665 codec_width_ *= 2;
5666 timestamp_ms += 1000 / kDefaultFramerate;
5667 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5668 WaitForEncodedFrame(timestamp_ms);
5669
5670 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5671 auto current_rate_settings =
5672 fake_encoder_.GetAndResetLastRateControlSettings();
5673 // Ensure we have actually reconfigured twice
5674 // The rate settings should have been set again even though
5675 // they haven't changed.
5676 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005677 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005678
5679 video_stream_encoder_->Stop();
5680}
5681
philipeld9cc8c02019-09-16 14:53:40 +02005682struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005683 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5684 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5685 MOCK_METHOD(void,
5686 RequestEncoderSwitch,
5687 (const webrtc::SdpVideoFormat& format),
5688 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005689};
5690
5691TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5692 constexpr int kDontCare = 100;
5693
5694 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5695 video_send_config_.encoder_settings.encoder_switch_request_callback =
5696 &switch_callback;
5697 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5698 encoder_config.codec_type = kVideoCodecVP8;
5699 webrtc::test::ScopedFieldTrials field_trial(
5700 "WebRTC-NetworkCondition-EncoderSwitch/"
5701 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5702 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5703
5704 // Reset encoder for new configuration to take effect.
5705 ConfigureEncoder(std::move(encoder_config));
5706
5707 // Send one frame to trigger ReconfigureEncoder.
5708 video_source_.IncomingCapturedFrame(
5709 CreateFrame(kDontCare, kDontCare, kDontCare));
5710
5711 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005712 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5713 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005714 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005715 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005716
Henrik Boström381d1092020-05-12 18:49:07 +02005717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005718 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5719 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5720 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005721 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005722 /*rtt_ms=*/0,
5723 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005724
5725 video_stream_encoder_->Stop();
5726}
5727
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005728TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5729 constexpr int kDontCare = 100;
5730
5731 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5732 video_send_config_.encoder_settings.encoder_switch_request_callback =
5733 &switch_callback;
5734 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5735 encoder_config.codec_type = kVideoCodecVP8;
5736 webrtc::test::ScopedFieldTrials field_trial(
5737 "WebRTC-NetworkCondition-EncoderSwitch/"
5738 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5739 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5740
5741 // Reset encoder for new configuration to take effect.
5742 ConfigureEncoder(std::move(encoder_config));
5743
5744 // Send one frame to trigger ReconfigureEncoder.
5745 video_source_.IncomingCapturedFrame(
5746 CreateFrame(kDontCare, kDontCare, kDontCare));
5747
5748 using Config = EncoderSwitchRequestCallback::Config;
5749 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5750 .Times(0);
5751
Henrik Boström381d1092020-05-12 18:49:07 +02005752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005753 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5754 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5755 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5756 /*fraction_lost=*/0,
5757 /*rtt_ms=*/0,
5758 /*cwnd_reduce_ratio=*/0);
5759
5760 video_stream_encoder_->Stop();
5761}
5762
philipeld9cc8c02019-09-16 14:53:40 +02005763TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5764 constexpr int kSufficientBitrateToNotDrop = 1000;
5765 constexpr int kHighRes = 500;
5766 constexpr int kLowRes = 100;
5767
5768 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5769 video_send_config_.encoder_settings.encoder_switch_request_callback =
5770 &switch_callback;
5771 webrtc::test::ScopedFieldTrials field_trial(
5772 "WebRTC-NetworkCondition-EncoderSwitch/"
5773 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5774 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5775 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5776 encoder_config.codec_type = kVideoCodecH264;
5777
5778 // Reset encoder for new configuration to take effect.
5779 ConfigureEncoder(std::move(encoder_config));
5780
5781 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5782 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5783 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005785 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5786 /*stable_target_bitrate=*/
5787 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5788 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005789 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005790 /*rtt_ms=*/0,
5791 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005792
5793 // Send one frame to trigger ReconfigureEncoder.
5794 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5795 WaitForEncodedFrame(1);
5796
5797 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005798 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5799 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005800 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005801 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005802
5803 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5804 WaitForEncodedFrame(2);
5805
5806 video_stream_encoder_->Stop();
5807}
5808
philipel9b058032020-02-10 11:30:00 +01005809TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5810 constexpr int kDontCare = 100;
5811 StrictMock<MockEncoderSelector> encoder_selector;
5812 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5813 &fake_encoder_, &encoder_selector);
5814 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5815
5816 // Reset encoder for new configuration to take effect.
5817 ConfigureEncoder(video_encoder_config_.Copy());
5818
5819 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5820
5821 video_source_.IncomingCapturedFrame(
5822 CreateFrame(kDontCare, kDontCare, kDontCare));
5823 video_stream_encoder_->Stop();
5824
5825 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5826 // to it's factory, so in order for the encoder instance in the
5827 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5828 // reset the |video_stream_encoder_| here.
5829 video_stream_encoder_.reset();
5830}
5831
5832TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5833 constexpr int kDontCare = 100;
5834
5835 NiceMock<MockEncoderSelector> encoder_selector;
5836 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5837 video_send_config_.encoder_settings.encoder_switch_request_callback =
5838 &switch_callback;
5839 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5840 &fake_encoder_, &encoder_selector);
5841 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5842
5843 // Reset encoder for new configuration to take effect.
5844 ConfigureEncoder(video_encoder_config_.Copy());
5845
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005846 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005847 .WillByDefault(Return(SdpVideoFormat("AV1")));
5848 EXPECT_CALL(switch_callback,
5849 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5850 Field(&SdpVideoFormat::name, "AV1"))));
5851
Henrik Boström381d1092020-05-12 18:49:07 +02005852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005853 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5854 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5855 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005856 /*fraction_lost=*/0,
5857 /*rtt_ms=*/0,
5858 /*cwnd_reduce_ratio=*/0);
5859
5860 video_stream_encoder_->Stop();
5861}
5862
5863TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5864 constexpr int kSufficientBitrateToNotDrop = 1000;
5865 constexpr int kDontCare = 100;
5866
5867 NiceMock<MockVideoEncoder> video_encoder;
5868 NiceMock<MockEncoderSelector> encoder_selector;
5869 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5870 video_send_config_.encoder_settings.encoder_switch_request_callback =
5871 &switch_callback;
5872 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5873 &video_encoder, &encoder_selector);
5874 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5875
5876 // Reset encoder for new configuration to take effect.
5877 ConfigureEncoder(video_encoder_config_.Copy());
5878
5879 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5880 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5881 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005882 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005883 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5884 /*stable_target_bitrate=*/
5885 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5886 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005887 /*fraction_lost=*/0,
5888 /*rtt_ms=*/0,
5889 /*cwnd_reduce_ratio=*/0);
5890
5891 ON_CALL(video_encoder, Encode(_, _))
5892 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5893 ON_CALL(encoder_selector, OnEncoderBroken())
5894 .WillByDefault(Return(SdpVideoFormat("AV2")));
5895
5896 rtc::Event encode_attempted;
5897 EXPECT_CALL(switch_callback,
5898 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5899 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5900 EXPECT_EQ(format.name, "AV2");
5901 encode_attempted.Set();
5902 });
5903
5904 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5905 encode_attempted.Wait(3000);
5906
5907 video_stream_encoder_->Stop();
5908
5909 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5910 // to it's factory, so in order for the encoder instance in the
5911 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5912 // reset the |video_stream_encoder_| here.
5913 video_stream_encoder_.reset();
5914}
5915
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005916TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005917 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005918 const int kFrameWidth = 320;
5919 const int kFrameHeight = 180;
5920
5921 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005922 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005924 /*target_bitrate=*/rate,
5925 /*stable_target_bitrate=*/rate,
5926 /*link_allocation=*/rate,
5927 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005928 /*rtt_ms=*/0,
5929 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005930
5931 // Insert a first video frame so that encoder gets configured.
5932 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5933 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5934 frame.set_rotation(kVideoRotation_270);
5935 video_source_.IncomingCapturedFrame(frame);
5936 WaitForEncodedFrame(timestamp_ms);
5937 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5938
5939 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005940 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005942 /*target_bitrate=*/new_stable_rate,
5943 /*stable_target_bitrate=*/new_stable_rate,
5944 /*link_allocation=*/rate,
5945 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005946 /*rtt_ms=*/0,
5947 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005948 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5949 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5950 video_stream_encoder_->Stop();
5951}
5952
5953TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005954 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005955 const int kFrameWidth = 320;
5956 const int kFrameHeight = 180;
5957
5958 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005959 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005961 /*target_bitrate=*/rate,
5962 /*stable_target_bitrate=*/rate,
5963 /*link_allocation=*/rate,
5964 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005965 /*rtt_ms=*/0,
5966 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005967
5968 // Insert a first video frame so that encoder gets configured.
5969 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5970 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5971 frame.set_rotation(kVideoRotation_270);
5972 video_source_.IncomingCapturedFrame(frame);
5973 WaitForEncodedFrame(timestamp_ms);
5974 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5975
5976 // Set a higher target rate without changing the link_allocation. Should not
5977 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005978 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005980 /*target_bitrate=*/rate,
5981 /*stable_target_bitrate=*/new_stable_rate,
5982 /*link_allocation=*/rate,
5983 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005984 /*rtt_ms=*/0,
5985 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005986 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5987 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5988 video_stream_encoder_->Stop();
5989}
5990
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005991TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5992 test::ScopedFieldTrials field_trials(
5993 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5994 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5995 const int kFramerateFps = 30;
5996 const int kWidth = 1920;
5997 const int kHeight = 1080;
5998 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5999 // Works on screenshare mode.
6000 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
6001 // We rely on the automatic resolution adaptation, but we handle framerate
6002 // adaptation manually by mocking the stats proxy.
6003 video_source_.set_adaptation_enabled(true);
6004
6005 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02006006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006007 DataRate::BitsPerSec(kTargetBitrateBps),
6008 DataRate::BitsPerSec(kTargetBitrateBps),
6009 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006010 video_stream_encoder_->SetSource(&video_source_,
6011 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006012 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006013
6014 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6015 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
6016
6017 // Pass enough frames with the full update to trigger animation detection.
6018 for (int i = 0; i < kNumFrames; ++i) {
6019 int64_t timestamp_ms =
6020 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
6021 frame.set_ntp_time_ms(timestamp_ms);
6022 frame.set_timestamp_us(timestamp_ms * 1000);
6023 video_source_.IncomingCapturedFrame(frame);
6024 WaitForEncodedFrame(timestamp_ms);
6025 }
6026
6027 // Resolution should be limited.
6028 rtc::VideoSinkWants expected;
6029 expected.max_framerate_fps = kFramerateFps;
6030 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006031 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006032
6033 // Pass one frame with no known update.
6034 // Resolution cap should be removed immediately.
6035 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
6036 frame.set_ntp_time_ms(timestamp_ms);
6037 frame.set_timestamp_us(timestamp_ms * 1000);
6038 frame.clear_update_rect();
6039
6040 video_source_.IncomingCapturedFrame(frame);
6041 WaitForEncodedFrame(timestamp_ms);
6042
6043 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006044 EXPECT_THAT(video_source_.sink_wants(),
6045 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006046
6047 video_stream_encoder_->Stop();
6048}
6049
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02006050TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
6051 const int kWidth = 720; // 540p adapted down.
6052 const int kHeight = 405;
6053 const int kNumFrames = 3;
6054 // Works on screenshare mode.
6055 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
6056 /*num_spatial_layers=*/2, /*screenshare=*/true);
6057
6058 video_source_.set_adaptation_enabled(true);
6059
6060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6061 DataRate::BitsPerSec(kTargetBitrateBps),
6062 DataRate::BitsPerSec(kTargetBitrateBps),
6063 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6064
6065 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6066
6067 // Pass enough frames with the full update to trigger animation detection.
6068 for (int i = 0; i < kNumFrames; ++i) {
6069 int64_t timestamp_ms =
6070 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
6071 frame.set_ntp_time_ms(timestamp_ms);
6072 frame.set_timestamp_us(timestamp_ms * 1000);
6073 video_source_.IncomingCapturedFrame(frame);
6074 WaitForEncodedFrame(timestamp_ms);
6075 }
6076
6077 video_stream_encoder_->Stop();
6078}
6079
perkj26091b12016-09-01 01:17:40 -07006080} // namespace webrtc