blob: 504b7b7b9cd684196b4c55e662baa21a71550406 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020029#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010030#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020031#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070032#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020034#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020035#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020036#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010037#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020038#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020039#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020042#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010043#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020044#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020045#include "system_wrappers/include/sleep.h"
46#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020060using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020061using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020062using ::testing::Ge;
63using ::testing::Gt;
64using ::testing::Le;
65using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010066using ::testing::Matcher;
67using ::testing::NiceMock;
68using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020069using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080070
perkj803d97f2016-11-01 11:45:46 -070071namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020072const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010073const int kQpLow = 1;
74const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020075const int kMinFramerateFps = 2;
76const int kMinBalancedFramerateFps = 7;
77const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080078const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010079const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020080const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kSimulcastTargetBitrateBps = 3150000;
82const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080083const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070084const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020085const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020086const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020087const VideoEncoder::ResolutionBitrateLimits
88 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
89const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080091
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020092uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
93 0x00, 0x00, 0x03, 0x03, 0xF4,
94 0x05, 0x03, 0xC7, 0xE0, 0x1B,
95 0x41, 0x10, 0x8D, 0x00};
96
perkj803d97f2016-11-01 11:45:46 -070097class TestBuffer : public webrtc::I420Buffer {
98 public:
99 TestBuffer(rtc::Event* event, int width, int height)
100 : I420Buffer(width, height), event_(event) {}
101
102 private:
103 friend class rtc::RefCountedObject<TestBuffer>;
104 ~TestBuffer() override {
105 if (event_)
106 event_->Set();
107 }
108 rtc::Event* const event_;
109};
110
Noah Richards51db4212019-06-12 06:59:12 -0700111// A fake native buffer that can't be converted to I420.
112class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
113 public:
114 FakeNativeBuffer(rtc::Event* event, int width, int height)
115 : event_(event), width_(width), height_(height) {}
116 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
117 int width() const override { return width_; }
118 int height() const override { return height_; }
119 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
120 return nullptr;
121 }
122
123 private:
124 friend class rtc::RefCountedObject<FakeNativeBuffer>;
125 ~FakeNativeBuffer() override {
126 if (event_)
127 event_->Set();
128 }
129 rtc::Event* const event_;
130 const int width_;
131 const int height_;
132};
133
Niels Möller7dc26b72017-12-06 10:27:48 +0100134class CpuOveruseDetectorProxy : public OveruseFrameDetector {
135 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200136 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
137 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200138 last_target_framerate_fps_(-1),
139 framerate_updated_event_(true /* manual_reset */,
140 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100141 virtual ~CpuOveruseDetectorProxy() {}
142
143 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200144 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100145 last_target_framerate_fps_ = framerate_fps;
146 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200147 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100148 }
149
150 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200151 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100152 return last_target_framerate_fps_;
153 }
154
Niels Möller4db138e2018-04-19 09:04:13 +0200155 CpuOveruseOptions GetOptions() { return options_; }
156
Henrik Boström381d1092020-05-12 18:49:07 +0200157 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
158
Niels Möller7dc26b72017-12-06 10:27:48 +0100159 private:
Markus Handella3765182020-07-08 13:13:32 +0200160 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100161 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200162 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100163};
164
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200165class FakeVideoSourceRestrictionsListener
166 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200167 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200168 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200169 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200170 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200171 RTC_DCHECK(was_restrictions_updated_);
172 }
173
174 rtc::Event* restrictions_updated_event() {
175 return &restrictions_updated_event_;
176 }
177
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200178 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200179 void OnVideoSourceRestrictionsUpdated(
180 VideoSourceRestrictions restrictions,
181 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200182 rtc::scoped_refptr<Resource> reason,
183 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200184 was_restrictions_updated_ = true;
185 restrictions_updated_event_.Set();
186 }
187
188 private:
189 bool was_restrictions_updated_;
190 rtc::Event restrictions_updated_event_;
191};
192
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200193auto WantsFps(Matcher<int> fps_matcher) {
194 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
195 fps_matcher);
196}
197
198auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
199 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
200 AllOf(max_pixel_matcher, Gt(0)));
201}
202
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200203auto ResolutionMax() {
204 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200205 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200206 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
207 Eq(absl::nullopt)));
208}
209
210auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200211 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200212}
213
214auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200215 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200216}
217
218auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200219 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200220}
221
222auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200223 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200224}
225
226auto FpsMaxResolutionMax() {
227 return AllOf(FpsMax(), ResolutionMax());
228}
229
230auto UnlimitedSinkWants() {
231 return AllOf(FpsUnlimited(), ResolutionMax());
232}
233
234auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
235 Matcher<int> fps_range_matcher;
236
237 if (last_frame_pixels <= 320 * 240) {
238 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200239 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200240 fps_range_matcher = AllOf(Ge(10), Le(15));
241 } else if (last_frame_pixels <= 640 * 480) {
242 fps_range_matcher = Ge(15);
243 } else {
244 fps_range_matcher = Eq(kDefaultFramerate);
245 }
246 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
247 fps_range_matcher);
248}
249
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200250auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
251 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
252 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
253}
254
255auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
256 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
257}
258
259auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
260 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
261}
262
263auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
264 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
265 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
266}
267
268auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
269 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
270 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
271}
272
273auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
274 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
275 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
276}
277
278auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
279 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
280 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
281}
282
mflodmancc3d4422017-08-03 08:27:51 -0700283class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700284 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200285 VideoStreamEncoderUnderTest(TimeController* time_controller,
286 TaskQueueFactory* task_queue_factory,
287 SendStatisticsProxy* stats_proxy,
288 const VideoStreamEncoderSettings& settings)
289 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100290 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200291 stats_proxy,
292 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200293 std::unique_ptr<OveruseFrameDetector>(
294 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100295 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100296 task_queue_factory),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200297 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200298 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200299 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200300 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200301 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200302 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200303 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200304 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100305 }
perkj803d97f2016-11-01 11:45:46 -0700306
Henrik Boström381d1092020-05-12 18:49:07 +0200307 void SetSourceAndWaitForRestrictionsUpdated(
308 rtc::VideoSourceInterface<VideoFrame>* source,
309 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200310 FakeVideoSourceRestrictionsListener listener;
311 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200312 SetSource(source, degradation_preference);
313 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200314 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200315 }
316
317 void SetSourceAndWaitForFramerateUpdated(
318 rtc::VideoSourceInterface<VideoFrame>* source,
319 const DegradationPreference& degradation_preference) {
320 overuse_detector_proxy_->framerate_updated_event()->Reset();
321 SetSource(source, degradation_preference);
322 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
323 }
324
325 void OnBitrateUpdatedAndWaitForManagedResources(
326 DataRate target_bitrate,
327 DataRate stable_target_bitrate,
328 DataRate link_allocation,
329 uint8_t fraction_lost,
330 int64_t round_trip_time_ms,
331 double cwnd_reduce_ratio) {
332 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
333 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
334 // Bitrate is updated on the encoder queue.
335 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200336 }
337
kthelgason2fc52542017-03-03 00:24:41 -0800338 // This is used as a synchronisation mechanism, to make sure that the
339 // encoder queue is not blocked before we start sending it frames.
340 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100341 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200342 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800343 ASSERT_TRUE(event.Wait(5000));
344 }
345
Henrik Boström91aa7322020-04-28 12:24:33 +0200346 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200347 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200348 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200349 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200350 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200351 event.Set();
352 });
353 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200354 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200355 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200356
Henrik Boström91aa7322020-04-28 12:24:33 +0200357 void TriggerCpuUnderuse() {
358 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200359 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200360 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200361 event.Set();
362 });
363 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200364 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200365 }
kthelgason876222f2016-11-29 01:44:11 -0800366
Henrik Boström91aa7322020-04-28 12:24:33 +0200367 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200368 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200369 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200370 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200371 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200372 event.Set();
373 });
374 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200375 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200376 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200377 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200378 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200379 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200380 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200381 event.Set();
382 });
383 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200384 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200385 }
386
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200387 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100388 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200389 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
390 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200391 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700392};
393
asapersson5f7226f2016-11-25 04:37:00 -0800394class VideoStreamFactory
395 : public VideoEncoderConfig::VideoStreamFactoryInterface {
396 public:
sprangfda496a2017-06-15 04:21:07 -0700397 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
398 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800399 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700400 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800401 }
402
403 private:
404 std::vector<VideoStream> CreateEncoderStreams(
405 int width,
406 int height,
407 const VideoEncoderConfig& encoder_config) override {
408 std::vector<VideoStream> streams =
409 test::CreateVideoStreams(width, height, encoder_config);
410 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100411 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700412 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800413 }
414 return streams;
415 }
sprangfda496a2017-06-15 04:21:07 -0700416
asapersson5f7226f2016-11-25 04:37:00 -0800417 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700418 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800419};
420
Noah Richards51db4212019-06-12 06:59:12 -0700421// Simulates simulcast behavior and makes highest stream resolutions divisible
422// by 4.
423class CroppingVideoStreamFactory
424 : public VideoEncoderConfig::VideoStreamFactoryInterface {
425 public:
426 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
427 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
428 EXPECT_GT(num_temporal_layers, 0u);
429 EXPECT_GT(framerate, 0);
430 }
431
432 private:
433 std::vector<VideoStream> CreateEncoderStreams(
434 int width,
435 int height,
436 const VideoEncoderConfig& encoder_config) override {
437 std::vector<VideoStream> streams = test::CreateVideoStreams(
438 width - width % 4, height - height % 4, encoder_config);
439 for (VideoStream& stream : streams) {
440 stream.num_temporal_layers = num_temporal_layers_;
441 stream.max_framerate = framerate_;
442 }
443 return streams;
444 }
445
446 const size_t num_temporal_layers_;
447 const int framerate_;
448};
449
sprangb1ca0732017-02-01 08:38:12 -0800450class AdaptingFrameForwarder : public test::FrameForwarder {
451 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200452 explicit AdaptingFrameForwarder(TimeController* time_controller)
453 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700454 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800455
456 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200457 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800458 adaptation_enabled_ = enabled;
459 }
460
asaperssonfab67072017-04-04 05:51:49 -0700461 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200462 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800463 return adaptation_enabled_;
464 }
465
asapersson09f05612017-05-15 23:40:18 -0700466 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200467 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700468 return last_wants_;
469 }
470
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200471 absl::optional<int> last_sent_width() const { return last_width_; }
472 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800473
sprangb1ca0732017-02-01 08:38:12 -0800474 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200475 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
476 time_controller_->AdvanceTime(TimeDelta::Millis(0));
477
sprangb1ca0732017-02-01 08:38:12 -0800478 int cropped_width = 0;
479 int cropped_height = 0;
480 int out_width = 0;
481 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700482 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200483 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
484 << "w=" << video_frame.width()
485 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700486 if (adapter_.AdaptFrameResolution(
487 video_frame.width(), video_frame.height(),
488 video_frame.timestamp_us() * 1000, &cropped_width,
489 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100490 VideoFrame adapted_frame =
491 VideoFrame::Builder()
492 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
493 nullptr, out_width, out_height))
494 .set_timestamp_rtp(99)
495 .set_timestamp_ms(99)
496 .set_rotation(kVideoRotation_0)
497 .build();
sprangc5d62e22017-04-02 23:53:04 -0700498 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100499 if (video_frame.has_update_rect()) {
500 adapted_frame.set_update_rect(
501 video_frame.update_rect().ScaleWithFrame(
502 video_frame.width(), video_frame.height(), 0, 0,
503 video_frame.width(), video_frame.height(), out_width,
504 out_height));
505 }
sprangc5d62e22017-04-02 23:53:04 -0700506 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800507 last_width_.emplace(adapted_frame.width());
508 last_height_.emplace(adapted_frame.height());
509 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200510 last_width_ = absl::nullopt;
511 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700512 }
sprangb1ca0732017-02-01 08:38:12 -0800513 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200514 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800515 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800516 last_width_.emplace(video_frame.width());
517 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800518 }
519 }
520
521 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
522 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200523 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200524 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100525 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200526 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800527 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200528
529 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800530 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200531 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
532 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200533 absl::optional<int> last_width_;
534 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800535};
sprangc5d62e22017-04-02 23:53:04 -0700536
Niels Möller213618e2018-07-24 09:29:58 +0200537// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700538class MockableSendStatisticsProxy : public SendStatisticsProxy {
539 public:
540 MockableSendStatisticsProxy(Clock* clock,
541 const VideoSendStream::Config& config,
542 VideoEncoderConfig::ContentType content_type)
543 : SendStatisticsProxy(clock, config, content_type) {}
544
545 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200546 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700547 if (mock_stats_)
548 return *mock_stats_;
549 return SendStatisticsProxy::GetStats();
550 }
551
Niels Möller213618e2018-07-24 09:29:58 +0200552 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200553 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200554 if (mock_stats_)
555 return mock_stats_->input_frame_rate;
556 return SendStatisticsProxy::GetInputFrameRate();
557 }
sprangc5d62e22017-04-02 23:53:04 -0700558 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200559 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700560 mock_stats_.emplace(stats);
561 }
562
563 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200564 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700565 mock_stats_.reset();
566 }
567
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200568 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
569 on_frame_dropped_ = std::move(callback);
570 }
571
sprangc5d62e22017-04-02 23:53:04 -0700572 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200573 void OnFrameDropped(DropReason reason) override {
574 SendStatisticsProxy::OnFrameDropped(reason);
575 if (on_frame_dropped_)
576 on_frame_dropped_(reason);
577 }
578
Markus Handella3765182020-07-08 13:13:32 +0200579 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200580 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200581 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700582};
583
sprang4847ae62017-06-27 07:06:52 -0700584class MockBitrateObserver : public VideoBitrateAllocationObserver {
585 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200586 MOCK_METHOD(void,
587 OnBitrateAllocationUpdated,
588 (const VideoBitrateAllocation&),
589 (override));
sprang4847ae62017-06-27 07:06:52 -0700590};
591
philipel9b058032020-02-10 11:30:00 +0100592class MockEncoderSelector
593 : public VideoEncoderFactory::EncoderSelectorInterface {
594 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200595 MOCK_METHOD(void,
596 OnCurrentEncoder,
597 (const SdpVideoFormat& format),
598 (override));
599 MOCK_METHOD(absl::optional<SdpVideoFormat>,
600 OnAvailableBitrate,
601 (const DataRate& rate),
602 (override));
603 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100604};
605
perkj803d97f2016-11-01 11:45:46 -0700606} // namespace
607
mflodmancc3d4422017-08-03 08:27:51 -0700608class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700609 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700611
mflodmancc3d4422017-08-03 08:27:51 -0700612 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700613 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700614 codec_width_(320),
615 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200616 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200617 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200618 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700619 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200620 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700621 video_send_config_,
622 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200623 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700624
625 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700626 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700627 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200628 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800629 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200630 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200631 video_send_config_.rtp.payload_name = "FAKE";
632 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700633
Per512ecb32016-09-23 15:52:06 +0200634 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200635 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700636 video_encoder_config.video_stream_factory =
637 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100638 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700639
640 // Framerate limit is specified by the VideoStreamFactory.
641 std::vector<VideoStream> streams =
642 video_encoder_config.video_stream_factory->CreateEncoderStreams(
643 codec_width_, codec_height_, video_encoder_config);
644 max_framerate_ = streams[0].max_framerate;
sprang4847ae62017-06-27 07:06:52 -0700645
Niels Möllerf1338562018-04-26 09:51:47 +0200646 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800647 }
648
Niels Möllerf1338562018-04-26 09:51:47 +0200649 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700650 if (video_stream_encoder_)
651 video_stream_encoder_->Stop();
652 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
654 video_send_config_.encoder_settings));
mflodmancc3d4422017-08-03 08:27:51 -0700655 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
656 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700657 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700658 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
659 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200660 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700661 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800662 }
663
664 void ResetEncoder(const std::string& payload_name,
665 size_t num_streams,
666 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700667 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700668 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200669 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800670
671 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200672 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800673 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100674 video_encoder_config.max_bitrate_bps =
675 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800676 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700677 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
678 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700679 video_encoder_config.content_type =
680 screenshare ? VideoEncoderConfig::ContentType::kScreen
681 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700682 if (payload_name == "VP9") {
683 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
684 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200685 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700686 video_encoder_config.encoder_specific_settings =
687 new rtc::RefCountedObject<
688 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
689 }
Niels Möllerf1338562018-04-26 09:51:47 +0200690 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700691 }
692
sprang57c2fff2017-01-16 06:24:02 -0800693 VideoFrame CreateFrame(int64_t ntp_time_ms,
694 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100695 VideoFrame frame =
696 VideoFrame::Builder()
697 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
698 destruction_event, codec_width_, codec_height_))
699 .set_timestamp_rtp(99)
700 .set_timestamp_ms(99)
701 .set_rotation(kVideoRotation_0)
702 .build();
sprang57c2fff2017-01-16 06:24:02 -0800703 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700704 return frame;
705 }
706
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100707 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
708 rtc::Event* destruction_event,
709 int offset_x) const {
710 VideoFrame frame =
711 VideoFrame::Builder()
712 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
713 destruction_event, codec_width_, codec_height_))
714 .set_timestamp_rtp(99)
715 .set_timestamp_ms(99)
716 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100717 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100718 .build();
719 frame.set_ntp_time_ms(ntp_time_ms);
720 return frame;
721 }
722
sprang57c2fff2017-01-16 06:24:02 -0800723 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100724 VideoFrame frame =
725 VideoFrame::Builder()
726 .set_video_frame_buffer(
727 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
728 .set_timestamp_rtp(99)
729 .set_timestamp_ms(99)
730 .set_rotation(kVideoRotation_0)
731 .build();
sprang57c2fff2017-01-16 06:24:02 -0800732 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700733 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700734 return frame;
735 }
736
Noah Richards51db4212019-06-12 06:59:12 -0700737 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
738 rtc::Event* destruction_event,
739 int width,
740 int height) const {
741 VideoFrame frame =
742 VideoFrame::Builder()
743 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
744 destruction_event, width, height))
745 .set_timestamp_rtp(99)
746 .set_timestamp_ms(99)
747 .set_rotation(kVideoRotation_0)
748 .build();
749 frame.set_ntp_time_ms(ntp_time_ms);
750 return frame;
751 }
752
753 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
754 rtc::Event* destruction_event) const {
755 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
756 codec_height_);
757 }
758
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100759 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
760 MockBitrateObserver bitrate_observer;
761 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
762
763 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
764 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100766 DataRate::BitsPerSec(kTargetBitrateBps),
767 DataRate::BitsPerSec(kTargetBitrateBps),
768 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100769
770 video_source_.IncomingCapturedFrame(
771 CreateFrame(1, codec_width_, codec_height_));
772 WaitForEncodedFrame(1);
773 }
774
sprang4847ae62017-06-27 07:06:52 -0700775 void WaitForEncodedFrame(int64_t expected_ntp_time) {
776 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200777 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700778 }
779
780 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
781 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200782 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700783 return ok;
784 }
785
786 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
787 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200788 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700789 }
790
791 void ExpectDroppedFrame() {
792 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200793 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700794 }
795
796 bool WaitForFrame(int64_t timeout_ms) {
797 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200798 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700799 return ok;
800 }
801
perkj26091b12016-09-01 01:17:40 -0700802 class TestEncoder : public test::FakeEncoder {
803 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200804 explicit TestEncoder(TimeController* time_controller)
805 : FakeEncoder(time_controller->GetClock()),
806 time_controller_(time_controller) {
807 RTC_DCHECK(time_controller_);
808 }
perkj26091b12016-09-01 01:17:40 -0700809
asaperssonfab67072017-04-04 05:51:49 -0700810 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200811 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700812 return config_;
813 }
814
815 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200816 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700817 block_next_encode_ = true;
818 }
819
Erik Språngaed30702018-11-05 12:57:17 +0100820 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200821 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100822 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100823 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100824 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100825 info.scaling_settings = VideoEncoder::ScalingSettings(
826 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100827 }
828 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100829 for (int i = 0; i < kMaxSpatialLayers; ++i) {
830 if (temporal_layers_supported_[i]) {
831 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
832 info.fps_allocation[i].resize(num_layers);
833 }
834 }
Erik Språngaed30702018-11-05 12:57:17 +0100835 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200836
837 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100838 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200839 info.apply_alignment_to_all_simulcast_layers =
840 apply_alignment_to_all_simulcast_layers_;
Erik Språngaed30702018-11-05 12:57:17 +0100841 return info;
kthelgason876222f2016-11-29 01:44:11 -0800842 }
843
Erik Språngb7cb7b52019-02-26 15:52:33 +0100844 int32_t RegisterEncodeCompleteCallback(
845 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200846 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100847 encoded_image_callback_ = callback;
848 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
849 }
850
perkjfa10b552016-10-02 23:45:26 -0700851 void ContinueEncode() { continue_encode_event_.Set(); }
852
853 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
854 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200855 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700856 EXPECT_EQ(timestamp_, timestamp);
857 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
858 }
859
kthelgason2fc52542017-03-03 00:24:41 -0800860 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200861 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800862 quality_scaling_ = b;
863 }
kthelgasonad9010c2017-02-14 00:46:51 -0800864
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100865 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200866 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100867 requested_resolution_alignment_ = requested_resolution_alignment;
868 }
869
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200870 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
871 MutexLock lock(&local_mutex_);
872 apply_alignment_to_all_simulcast_layers_ = b;
873 }
874
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100875 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200876 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100877 is_hardware_accelerated_ = is_hardware_accelerated;
878 }
879
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100880 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
881 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200882 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100883 temporal_layers_supported_[spatial_idx] = supported;
884 }
885
Sergey Silkin6456e352019-07-08 17:56:40 +0200886 void SetResolutionBitrateLimits(
887 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200888 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200889 resolution_bitrate_limits_ = thresholds;
890 }
891
sprangfe627f32017-03-29 08:24:59 -0700892 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200893 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700894 force_init_encode_failed_ = force_failure;
895 }
896
Niels Möller6bb5ab92019-01-11 11:11:10 +0100897 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200898 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100899 rate_factor_ = rate_factor;
900 }
901
Erik Språngd7329ca2019-02-21 21:19:53 +0100902 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200903 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100904 return last_framerate_;
905 }
906
Erik Språngd7329ca2019-02-21 21:19:53 +0100907 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200908 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100909 return last_update_rect_;
910 }
911
Niels Möller87e2d782019-03-07 10:18:23 +0100912 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200913 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100914 return last_frame_types_;
915 }
916
917 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100918 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100919 keyframe ? VideoFrameType::kVideoFrameKey
920 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100921 {
Markus Handella3765182020-07-08 13:13:32 +0200922 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100923 last_frame_types_ = frame_type;
924 }
Niels Möllerb859b322019-03-07 12:40:01 +0100925 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100926 }
927
Erik Språngb7cb7b52019-02-26 15:52:33 +0100928 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200929 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200930 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100931 }
932
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200933 void SetEncodedImageData(
934 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200936 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200937 }
938
Erik Språngd7329ca2019-02-21 21:19:53 +0100939 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200940 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100941 expect_null_frame_ = true;
942 }
943
Erik Språng5056af02019-09-02 15:53:11 +0200944 absl::optional<VideoEncoder::RateControlParameters>
945 GetAndResetLastRateControlSettings() {
946 auto settings = last_rate_control_settings_;
947 last_rate_control_settings_.reset();
948 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100949 }
950
Sergey Silkin5ee69672019-07-02 14:18:34 +0200951 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200952 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200953 return num_encoder_initializations_;
954 }
955
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200956 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200957 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200958 return num_set_rates_;
959 }
960
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200961 VideoCodec video_codec() const {
962 MutexLock lock(&local_mutex_);
963 return video_codec_;
964 }
965
perkjfa10b552016-10-02 23:45:26 -0700966 private:
perkj26091b12016-09-01 01:17:40 -0700967 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100968 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700969 bool block_encode;
970 {
Markus Handella3765182020-07-08 13:13:32 +0200971 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100972 if (expect_null_frame_) {
973 EXPECT_EQ(input_image.timestamp(), 0u);
974 EXPECT_EQ(input_image.width(), 1);
975 last_frame_types_ = *frame_types;
976 expect_null_frame_ = false;
977 } else {
978 EXPECT_GT(input_image.timestamp(), timestamp_);
979 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
980 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
981 }
perkj26091b12016-09-01 01:17:40 -0700982
983 timestamp_ = input_image.timestamp();
984 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700985 last_input_width_ = input_image.width();
986 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700987 block_encode = block_next_encode_;
988 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100989 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100990 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700991 }
Niels Möllerb859b322019-03-07 12:40:01 +0100992 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700993 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700994 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200995
perkj26091b12016-09-01 01:17:40 -0700996 return result;
997 }
998
Danil Chapovalov2549f172020-08-12 17:30:36 +0200999 CodecSpecificInfo EncodeHook(EncodedImage& encoded_image) override {
1000 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001001 {
1002 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001003 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001004 }
1005 MutexLock lock(&local_mutex_);
1006 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001007 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001008 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001009 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001010 }
1011
sprangfe627f32017-03-29 08:24:59 -07001012 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001013 const Settings& settings) override {
1014 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001015
Markus Handella3765182020-07-08 13:13:32 +02001016 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001017 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001018
1019 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001020 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001021
Erik Språng82fad3d2018-03-21 09:57:23 +01001022 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001023 // Simulate setting up temporal layers, in order to validate the life
1024 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001025 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001026 frame_buffer_controller_ =
1027 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001028 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001029 if (force_init_encode_failed_) {
1030 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001031 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001032 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001033
Erik Språngb7cb7b52019-02-26 15:52:33 +01001034 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001035 return res;
1036 }
1037
Erik Språngb7cb7b52019-02-26 15:52:33 +01001038 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001039 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001040 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1041 initialized_ = EncoderState::kUninitialized;
1042 return FakeEncoder::Release();
1043 }
1044
Erik Språng16cb8f52019-04-12 13:59:09 +02001045 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001046 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001047 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001048 VideoBitrateAllocation adjusted_rate_allocation;
1049 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1050 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001051 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001052 adjusted_rate_allocation.SetBitrate(
1053 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001054 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001055 rate_factor_));
1056 }
1057 }
1058 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001059 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001060 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001061 RateControlParameters adjusted_paramters = parameters;
1062 adjusted_paramters.bitrate = adjusted_rate_allocation;
1063 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001064 }
1065
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001066 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001067 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001068 enum class EncoderState {
1069 kUninitialized,
1070 kInitializationFailed,
1071 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001072 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1073 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001074 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001075 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1076 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1077 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1078 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1079 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1080 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001081 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1082 false;
Markus Handella3765182020-07-08 13:13:32 +02001083 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001084 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1085 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001086 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001087 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001088 absl::optional<bool>
1089 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001090 local_mutex_);
1091 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1092 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1093 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001094 absl::optional<VideoEncoder::RateControlParameters>
1095 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001096 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1097 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001098 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001099 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001100 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1101 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001102 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001103 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001104 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001105 RTC_GUARDED_BY(local_mutex_);
1106 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001107 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001108 };
1109
mflodmancc3d4422017-08-03 08:27:51 -07001110 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001111 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001112 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1113 : time_controller_(time_controller), test_encoder_(test_encoder) {
1114 RTC_DCHECK(time_controller_);
1115 }
perkj26091b12016-09-01 01:17:40 -07001116
perkj26091b12016-09-01 01:17:40 -07001117 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001118 EXPECT_TRUE(
1119 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1120 }
1121
1122 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1123 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001124 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001125 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001126 return false;
perkj26091b12016-09-01 01:17:40 -07001127 {
Markus Handella3765182020-07-08 13:13:32 +02001128 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001129 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001130 }
1131 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001132 return true;
perkj26091b12016-09-01 01:17:40 -07001133 }
1134
sprangb1ca0732017-02-01 08:38:12 -08001135 void WaitForEncodedFrame(uint32_t expected_width,
1136 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001137 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001138 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001139 }
1140
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001141 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001142 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001143 uint32_t width = 0;
1144 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001145 {
Markus Handella3765182020-07-08 13:13:32 +02001146 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001147 width = last_width_;
1148 height = last_height_;
1149 }
1150 EXPECT_EQ(expected_height, height);
1151 EXPECT_EQ(expected_width, width);
1152 }
1153
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001154 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1155 VideoRotation rotation;
1156 {
Markus Handella3765182020-07-08 13:13:32 +02001157 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001158 rotation = last_rotation_;
1159 }
1160 EXPECT_EQ(expected_rotation, rotation);
1161 }
1162
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001163 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001164
sprangc5d62e22017-04-02 23:53:04 -07001165 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001166 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1167 bool ret = encoded_frame_event_.Wait(timeout_ms);
1168 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1169 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001170 }
1171
perkj26091b12016-09-01 01:17:40 -07001172 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001173 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001174 expect_frames_ = false;
1175 }
1176
asaperssonfab67072017-04-04 05:51:49 -07001177 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001178 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001179 return number_of_reconfigurations_;
1180 }
1181
asaperssonfab67072017-04-04 05:51:49 -07001182 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001183 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001184 return min_transmit_bitrate_bps_;
1185 }
1186
Erik Språngd7329ca2019-02-21 21:19:53 +01001187 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001188 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001189 num_expected_layers_ = num_layers;
1190 }
1191
Erik Språngb7cb7b52019-02-26 15:52:33 +01001192 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001193 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001194 return last_capture_time_ms_;
1195 }
1196
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001197 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001198 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001199 return std::move(last_encoded_image_data_);
1200 }
1201
perkj26091b12016-09-01 01:17:40 -07001202 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001203 Result OnEncodedImage(
1204 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001205 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001206 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001207 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001208 last_encoded_image_data_ = std::vector<uint8_t>(
1209 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001210 uint32_t timestamp = encoded_image.Timestamp();
1211 if (last_timestamp_ != timestamp) {
1212 num_received_layers_ = 1;
1213 } else {
1214 ++num_received_layers_;
1215 }
1216 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001217 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001218 last_width_ = encoded_image._encodedWidth;
1219 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001220 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001221 if (num_received_layers_ == num_expected_layers_) {
1222 encoded_frame_event_.Set();
1223 }
sprangb1ca0732017-02-01 08:38:12 -08001224 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001225 }
1226
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001227 void OnEncoderConfigurationChanged(
1228 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001229 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001230 VideoEncoderConfig::ContentType content_type,
1231 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001232 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001233 ++number_of_reconfigurations_;
1234 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1235 }
1236
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001237 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001238 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001239 TestEncoder* test_encoder_;
1240 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001241 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001242 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001243 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001244 uint32_t last_height_ = 0;
1245 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001246 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001247 size_t num_expected_layers_ = 1;
1248 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001249 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001250 int number_of_reconfigurations_ = 0;
1251 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001252 };
1253
Sergey Silkin5ee69672019-07-02 14:18:34 +02001254 class VideoBitrateAllocatorProxyFactory
1255 : public VideoBitrateAllocatorFactory {
1256 public:
1257 VideoBitrateAllocatorProxyFactory()
1258 : bitrate_allocator_factory_(
1259 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1260
1261 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1262 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001263 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001264 codec_config_ = codec;
1265 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1266 }
1267
1268 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001269 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001270 return codec_config_;
1271 }
1272
1273 private:
1274 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1275
Markus Handella3765182020-07-08 13:13:32 +02001276 mutable Mutex mutex_;
1277 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001278 };
1279
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001280 Clock* clock() { return time_controller_.GetClock(); }
1281 void AdvanceTime(TimeDelta duration) {
1282 time_controller_.AdvanceTime(duration);
1283 }
1284
1285 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1286
1287 protected:
1288 virtual TaskQueueFactory* GetTaskQueueFactory() {
1289 return time_controller_.GetTaskQueueFactory();
1290 }
1291
1292 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001293 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001294 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001295 int codec_width_;
1296 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001297 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001298 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001299 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001300 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001301 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001302 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001303 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001304 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001305};
1306
mflodmancc3d4422017-08-03 08:27:51 -07001307TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001308 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001309 DataRate::BitsPerSec(kTargetBitrateBps),
1310 DataRate::BitsPerSec(kTargetBitrateBps),
1311 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001312 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001313 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001314 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001315 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001316 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001317}
1318
mflodmancc3d4422017-08-03 08:27:51 -07001319TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001320 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001321 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001322 // The encoder will cache up to one frame for a short duration. Adding two
1323 // frames means that the first frame will be dropped and the second frame will
1324 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001325 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001326 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001327 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001328 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001329
Henrik Boström381d1092020-05-12 18:49:07 +02001330 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001331 DataRate::BitsPerSec(kTargetBitrateBps),
1332 DataRate::BitsPerSec(kTargetBitrateBps),
1333 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001334
Sebastian Janssona3177052018-04-10 13:05:49 +02001335 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001336 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001337 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1338
1339 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001340 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001341}
1342
mflodmancc3d4422017-08-03 08:27:51 -07001343TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001344 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001345 DataRate::BitsPerSec(kTargetBitrateBps),
1346 DataRate::BitsPerSec(kTargetBitrateBps),
1347 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001348 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001349 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001350
Henrik Boström381d1092020-05-12 18:49:07 +02001351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1352 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1353 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001354 // The encoder will cache up to one frame for a short duration. Adding two
1355 // frames means that the first frame will be dropped and the second frame will
1356 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001357 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001358 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001359
Henrik Boström381d1092020-05-12 18:49:07 +02001360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001361 DataRate::BitsPerSec(kTargetBitrateBps),
1362 DataRate::BitsPerSec(kTargetBitrateBps),
1363 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001364 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001365 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1366 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001367 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001368}
1369
mflodmancc3d4422017-08-03 08:27:51 -07001370TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001371 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001372 DataRate::BitsPerSec(kTargetBitrateBps),
1373 DataRate::BitsPerSec(kTargetBitrateBps),
1374 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001375 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001376 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001377
1378 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001379 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001380
perkja49cbd32016-09-16 07:53:41 -07001381 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001382 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001383 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001384}
1385
mflodmancc3d4422017-08-03 08:27:51 -07001386TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001387 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001388 DataRate::BitsPerSec(kTargetBitrateBps),
1389 DataRate::BitsPerSec(kTargetBitrateBps),
1390 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001391
perkja49cbd32016-09-16 07:53:41 -07001392 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001393 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001394
mflodmancc3d4422017-08-03 08:27:51 -07001395 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001396 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001397 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001398 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1399 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001400}
1401
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001402class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1403 public:
1404 VideoStreamEncoderBlockedTest() {}
1405
1406 TaskQueueFactory* GetTaskQueueFactory() override {
1407 return task_queue_factory_.get();
1408 }
1409
1410 private:
1411 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1412 CreateDefaultTaskQueueFactory();
1413};
1414
1415TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001417 DataRate::BitsPerSec(kTargetBitrateBps),
1418 DataRate::BitsPerSec(kTargetBitrateBps),
1419 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001420
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001421 int dropped_count = 0;
1422 stats_proxy_->SetDroppedFrameCallback(
1423 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1424 ++dropped_count;
1425 });
1426
perkj26091b12016-09-01 01:17:40 -07001427 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001428 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001429 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001430 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1431 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001432 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1433 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001434 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001435 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001436
mflodmancc3d4422017-08-03 08:27:51 -07001437 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001438
1439 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001440}
1441
Noah Richards51db4212019-06-12 06:59:12 -07001442TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001444 DataRate::BitsPerSec(kTargetBitrateBps),
1445 DataRate::BitsPerSec(kTargetBitrateBps),
1446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001447
1448 rtc::Event frame_destroyed_event;
1449 video_source_.IncomingCapturedFrame(
1450 CreateFakeNativeFrame(1, &frame_destroyed_event));
1451 ExpectDroppedFrame();
1452 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1453 video_stream_encoder_->Stop();
1454}
1455
1456TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1457 // Use the cropping factory.
1458 video_encoder_config_.video_stream_factory =
1459 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1460 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1461 kMaxPayloadLength);
1462 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1463
1464 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001465 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001466 DataRate::BitsPerSec(kTargetBitrateBps),
1467 DataRate::BitsPerSec(kTargetBitrateBps),
1468 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001469 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1470 WaitForEncodedFrame(1);
1471 // The encoder will have been configured once.
1472 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1473 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1474 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1475
1476 // Now send in a fake frame that needs to be cropped as the width/height
1477 // aren't divisible by 4 (see CreateEncoderStreams above).
1478 rtc::Event frame_destroyed_event;
1479 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1480 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1481 ExpectDroppedFrame();
1482 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1483 video_stream_encoder_->Stop();
1484}
1485
Ying Wang9b881ab2020-02-07 14:29:32 +01001486TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001488 DataRate::BitsPerSec(kTargetBitrateBps),
1489 DataRate::BitsPerSec(kTargetBitrateBps),
1490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001491 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1492 WaitForEncodedFrame(1);
1493
Henrik Boström381d1092020-05-12 18:49:07 +02001494 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001495 DataRate::BitsPerSec(kTargetBitrateBps),
1496 DataRate::BitsPerSec(kTargetBitrateBps),
1497 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001498 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1499 // frames. Adding two frames means that the first frame will be dropped and
1500 // the second frame will be sent to the encoder.
1501 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1502 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1503 WaitForEncodedFrame(3);
1504 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1505 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1506 WaitForEncodedFrame(5);
1507 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1508 video_stream_encoder_->Stop();
1509}
1510
mflodmancc3d4422017-08-03 08:27:51 -07001511TEST_F(VideoStreamEncoderTest,
1512 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001514 DataRate::BitsPerSec(kTargetBitrateBps),
1515 DataRate::BitsPerSec(kTargetBitrateBps),
1516 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001517 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001518
1519 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001520 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001521 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001522 // The encoder will have been configured once when the first frame is
1523 // received.
1524 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001525
1526 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001527 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001528 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001529 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001530 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001531
1532 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001533 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001534 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001535 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001536 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001537
mflodmancc3d4422017-08-03 08:27:51 -07001538 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001539}
1540
mflodmancc3d4422017-08-03 08:27:51 -07001541TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001543 DataRate::BitsPerSec(kTargetBitrateBps),
1544 DataRate::BitsPerSec(kTargetBitrateBps),
1545 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001546
1547 // Capture a frame and wait for it to synchronize with the encoder thread.
1548 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001549 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001550 // The encoder will have been configured once.
1551 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001552 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1553 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1554
1555 codec_width_ *= 2;
1556 codec_height_ *= 2;
1557 // Capture a frame with a higher resolution and wait for it to synchronize
1558 // with the encoder thread.
1559 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001560 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001561 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1562 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001563 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001564
mflodmancc3d4422017-08-03 08:27:51 -07001565 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001566}
1567
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001568TEST_F(VideoStreamEncoderTest,
1569 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001571 DataRate::BitsPerSec(kTargetBitrateBps),
1572 DataRate::BitsPerSec(kTargetBitrateBps),
1573 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001574
1575 // Capture a frame and wait for it to synchronize with the encoder thread.
1576 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1577 WaitForEncodedFrame(1);
1578
1579 VideoEncoderConfig video_encoder_config;
1580 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1581 // Changing the max payload data length recreates encoder.
1582 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1583 kMaxPayloadLength / 2);
1584
1585 // Capture a frame and wait for it to synchronize with the encoder thread.
1586 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1587 WaitForEncodedFrame(2);
1588 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1589
1590 video_stream_encoder_->Stop();
1591}
1592
Sergey Silkin5ee69672019-07-02 14:18:34 +02001593TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001595 DataRate::BitsPerSec(kTargetBitrateBps),
1596 DataRate::BitsPerSec(kTargetBitrateBps),
1597 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001598
1599 VideoEncoderConfig video_encoder_config;
1600 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1601 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1602 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1603 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1604 kMaxPayloadLength);
1605
1606 // Capture a frame and wait for it to synchronize with the encoder thread.
1607 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1608 WaitForEncodedFrame(1);
1609 // The encoder will have been configured once when the first frame is
1610 // received.
1611 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1612 EXPECT_EQ(kTargetBitrateBps,
1613 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1614 EXPECT_EQ(kStartBitrateBps,
1615 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1616
Sergey Silkin6456e352019-07-08 17:56:40 +02001617 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1618 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001619 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1620 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1621 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1622 kMaxPayloadLength);
1623
1624 // Capture a frame and wait for it to synchronize with the encoder thread.
1625 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1626 WaitForEncodedFrame(2);
1627 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1628 // Bitrate limits have changed - rate allocator should be reconfigured,
1629 // encoder should not be reconfigured.
1630 EXPECT_EQ(kTargetBitrateBps * 2,
1631 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1632 EXPECT_EQ(kStartBitrateBps * 2,
1633 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1634 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1635
1636 video_stream_encoder_->Stop();
1637}
1638
Sergey Silkin6456e352019-07-08 17:56:40 +02001639TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001640 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001642 DataRate::BitsPerSec(kTargetBitrateBps),
1643 DataRate::BitsPerSec(kTargetBitrateBps),
1644 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001645
Sergey Silkincd02eba2020-01-20 14:48:40 +01001646 const uint32_t kMinEncBitrateKbps = 100;
1647 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001648 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001649 /*frame_size_pixels=*/codec_width_ * codec_height_,
1650 /*min_start_bitrate_bps=*/0,
1651 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1652 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001653 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1654
Sergey Silkincd02eba2020-01-20 14:48:40 +01001655 VideoEncoderConfig video_encoder_config;
1656 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1657 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1658 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1659 (kMinEncBitrateKbps + 1) * 1000;
1660 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1661 kMaxPayloadLength);
1662
1663 // When both encoder and app provide bitrate limits, the intersection of
1664 // provided sets should be used.
1665 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1666 WaitForEncodedFrame(1);
1667 EXPECT_EQ(kMaxEncBitrateKbps,
1668 bitrate_allocator_factory_.codec_config().maxBitrate);
1669 EXPECT_EQ(kMinEncBitrateKbps + 1,
1670 bitrate_allocator_factory_.codec_config().minBitrate);
1671
1672 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1673 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1674 (kMinEncBitrateKbps - 1) * 1000;
1675 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1676 kMaxPayloadLength);
1677 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001678 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001679 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001680 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001681 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001682 bitrate_allocator_factory_.codec_config().minBitrate);
1683
Sergey Silkincd02eba2020-01-20 14:48:40 +01001684 video_stream_encoder_->Stop();
1685}
1686
1687TEST_F(VideoStreamEncoderTest,
1688 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001690 DataRate::BitsPerSec(kTargetBitrateBps),
1691 DataRate::BitsPerSec(kTargetBitrateBps),
1692 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001693
1694 const uint32_t kMinAppBitrateKbps = 100;
1695 const uint32_t kMaxAppBitrateKbps = 200;
1696 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1697 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1698 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1699 /*frame_size_pixels=*/codec_width_ * codec_height_,
1700 /*min_start_bitrate_bps=*/0,
1701 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1702 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1703 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1704
1705 VideoEncoderConfig video_encoder_config;
1706 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1707 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1708 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1709 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001710 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1711 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001712
Sergey Silkincd02eba2020-01-20 14:48:40 +01001713 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1714 WaitForEncodedFrame(1);
1715 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001716 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001717 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001718 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001719
1720 video_stream_encoder_->Stop();
1721}
1722
1723TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001724 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001726 DataRate::BitsPerSec(kTargetBitrateBps),
1727 DataRate::BitsPerSec(kTargetBitrateBps),
1728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001729
1730 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001731 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001732 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001733 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001734 fake_encoder_.SetResolutionBitrateLimits(
1735 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1736
1737 VideoEncoderConfig video_encoder_config;
1738 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1739 video_encoder_config.max_bitrate_bps = 0;
1740 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1741 kMaxPayloadLength);
1742
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001743 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001744 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1745 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001746 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1747 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001748 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1749 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1750
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001751 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001752 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1753 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001754 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1755 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001756 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1757 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1758
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001759 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001760 // encoder for 360p should be used.
1761 video_source_.IncomingCapturedFrame(
1762 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1763 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001764 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1765 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001766 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1767 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1768
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001769 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001770 // ignored.
1771 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1772 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001773 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1774 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001775 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1776 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001777 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1778 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001779 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1780 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1781
1782 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1783 // for 270p should be used.
1784 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1785 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001786 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1787 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001788 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1789 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1790
1791 video_stream_encoder_->Stop();
1792}
1793
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001794TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001796 DataRate::BitsPerSec(kTargetBitrateBps),
1797 DataRate::BitsPerSec(kTargetBitrateBps),
1798 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001799
1800 VideoEncoderConfig video_encoder_config;
1801 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1802 video_encoder_config.max_bitrate_bps = 0;
1803 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1804 kMaxPayloadLength);
1805
1806 // Encode 720p frame to get the default encoder target bitrate.
1807 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1808 WaitForEncodedFrame(1);
1809 const uint32_t kDefaultTargetBitrateFor720pKbps =
1810 bitrate_allocator_factory_.codec_config()
1811 .simulcastStream[0]
1812 .targetBitrate;
1813
1814 // Set the max recommended encoder bitrate to something lower than the default
1815 // target bitrate.
1816 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1817 1280 * 720, 10 * 1000, 10 * 1000,
1818 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1819 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1820
1821 // Change resolution to trigger encoder reinitialization.
1822 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1823 WaitForEncodedFrame(2);
1824 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1825 WaitForEncodedFrame(3);
1826
1827 // Ensure the target bitrate is capped by the max bitrate.
1828 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1829 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1830 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1831 .simulcastStream[0]
1832 .targetBitrate *
1833 1000,
1834 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1835
1836 video_stream_encoder_->Stop();
1837}
1838
mflodmancc3d4422017-08-03 08:27:51 -07001839TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001840 EXPECT_TRUE(video_source_.has_sinks());
1841 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001843 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001844 EXPECT_FALSE(video_source_.has_sinks());
1845 EXPECT_TRUE(new_video_source.has_sinks());
1846
mflodmancc3d4422017-08-03 08:27:51 -07001847 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001848}
1849
mflodmancc3d4422017-08-03 08:27:51 -07001850TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001851 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001852 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001853 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001854 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001855}
1856
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001857class ResolutionAlignmentTest
1858 : public VideoStreamEncoderTest,
1859 public ::testing::WithParamInterface<
1860 ::testing::tuple<int, std::vector<double>>> {
1861 public:
1862 ResolutionAlignmentTest()
1863 : requested_alignment_(::testing::get<0>(GetParam())),
1864 scale_factors_(::testing::get<1>(GetParam())) {}
1865
1866 protected:
1867 const int requested_alignment_;
1868 const std::vector<double> scale_factors_;
1869};
1870
1871INSTANTIATE_TEST_SUITE_P(
1872 AlignmentAndScaleFactors,
1873 ResolutionAlignmentTest,
1874 ::testing::Combine(
1875 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
1876 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
1877 std::vector<double>{-1.0, -1.0},
1878 std::vector<double>{-1.0, -1.0, -1.0},
1879 std::vector<double>{4.0, 2.0, 1.0},
1880 std::vector<double>{9999.0, -1.0, 1.0},
1881 std::vector<double>{3.99, 2.01, 1.0},
1882 std::vector<double>{4.9, 1.7, 1.25},
1883 std::vector<double>{10.0, 4.0, 3.0},
1884 std::vector<double>{1.75, 3.5},
1885 std::vector<double>{1.5, 2.5},
1886 std::vector<double>{1.3, 1.0})));
1887
1888TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
1889 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001890 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001891 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
1892 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
1893
1894 // Fill config with the scaling factor by which to reduce encoding size.
1895 const int num_streams = scale_factors_.size();
1896 VideoEncoderConfig config;
1897 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
1898 for (int i = 0; i < num_streams; ++i) {
1899 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
1900 }
1901 config.video_stream_factory =
1902 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
1903 "VP8", /*max qp*/ 56, /*screencast*/ false,
1904 /*screenshare enabled*/ false);
1905 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
1906
Henrik Boström381d1092020-05-12 18:49:07 +02001907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001908 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
1909 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
1910 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
1911 // Wait for all layers before triggering event.
1912 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001913
1914 // On the 1st frame, we should have initialized the encoder and
1915 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001916 int64_t timestamp_ms = kFrameIntervalMs;
1917 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
1918 WaitForEncodedFrame(timestamp_ms);
1919 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001920
1921 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1922 // (It's up the to the encoder to potentially drop the previous frame,
1923 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001924 timestamp_ms += kFrameIntervalMs;
1925 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
1926 WaitForEncodedFrame(timestamp_ms);
1927 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
1928
1929 VideoCodec codec = fake_encoder_.video_codec();
1930 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
1931 // Frame size should be a multiple of the requested alignment.
1932 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
1933 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
1934 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
1935 // Aspect ratio should match.
1936 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
1937 codec.height * codec.simulcastStream[i].width);
1938 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001939
1940 video_stream_encoder_->Stop();
1941}
1942
Jonathan Yubc771b72017-12-08 17:04:29 -08001943TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1944 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001945 const int kWidth = 1280;
1946 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001947
1948 // We rely on the automatic resolution adaptation, but we handle framerate
1949 // adaptation manually by mocking the stats proxy.
1950 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001951
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001952 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001953 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001954 DataRate::BitsPerSec(kTargetBitrateBps),
1955 DataRate::BitsPerSec(kTargetBitrateBps),
1956 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001957 video_stream_encoder_->SetSource(&video_source_,
1958 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001959 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001960 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001961 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001962 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1963
Jonathan Yubc771b72017-12-08 17:04:29 -08001964 // Adapt down as far as possible.
1965 rtc::VideoSinkWants last_wants;
1966 int64_t t = 1;
1967 int loop_count = 0;
1968 do {
1969 ++loop_count;
1970 last_wants = video_source_.sink_wants();
1971
1972 // Simulate the framerate we've been asked to adapt to.
1973 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1974 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1975 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1976 mock_stats.input_frame_rate = fps;
1977 stats_proxy_->SetMockStats(mock_stats);
1978
1979 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1980 sink_.WaitForEncodedFrame(t);
1981 t += frame_interval_ms;
1982
mflodmancc3d4422017-08-03 08:27:51 -07001983 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001984 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001985 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001986 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1987 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001988 } while (video_source_.sink_wants().max_pixel_count <
1989 last_wants.max_pixel_count ||
1990 video_source_.sink_wants().max_framerate_fps <
1991 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001992
Jonathan Yubc771b72017-12-08 17:04:29 -08001993 // Verify that we've adapted all the way down.
1994 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001995 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001996 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1997 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001998 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001999 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2000 *video_source_.last_sent_height());
2001 EXPECT_EQ(kMinBalancedFramerateFps,
2002 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002003
Jonathan Yubc771b72017-12-08 17:04:29 -08002004 // Adapt back up the same number of times we adapted down.
2005 for (int i = 0; i < loop_count - 1; ++i) {
2006 last_wants = video_source_.sink_wants();
2007
2008 // Simulate the framerate we've been asked to adapt to.
2009 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2010 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2011 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2012 mock_stats.input_frame_rate = fps;
2013 stats_proxy_->SetMockStats(mock_stats);
2014
2015 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2016 sink_.WaitForEncodedFrame(t);
2017 t += frame_interval_ms;
2018
Henrik Boström91aa7322020-04-28 12:24:33 +02002019 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002020 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002021 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002022 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2023 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002024 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2025 last_wants.max_pixel_count ||
2026 video_source_.sink_wants().max_framerate_fps >
2027 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002028 }
2029
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002030 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002031 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002032 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002033 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2034 EXPECT_EQ((loop_count - 1) * 2,
2035 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002036
mflodmancc3d4422017-08-03 08:27:51 -07002037 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002038}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002039
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002040TEST_F(VideoStreamEncoderTest,
2041 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2042 video_stream_encoder_->OnBitrateUpdated(
2043 DataRate::BitsPerSec(kTargetBitrateBps),
2044 DataRate::BitsPerSec(kTargetBitrateBps),
2045 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002046 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002047
2048 const int kFrameWidth = 1280;
2049 const int kFrameHeight = 720;
2050
2051 int64_t ntp_time = kFrameIntervalMs;
2052
2053 // Force an input frame rate to be available, or the adaptation call won't
2054 // know what framerate to adapt form.
2055 const int kInputFps = 30;
2056 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2057 stats.input_frame_rate = kInputFps;
2058 stats_proxy_->SetMockStats(stats);
2059
2060 video_source_.set_adaptation_enabled(true);
2061 video_stream_encoder_->SetSource(
2062 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002063 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002064 video_source_.IncomingCapturedFrame(
2065 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2066 sink_.WaitForEncodedFrame(ntp_time);
2067 ntp_time += kFrameIntervalMs;
2068
2069 // Trigger CPU overuse.
2070 video_stream_encoder_->TriggerCpuOveruse();
2071 video_source_.IncomingCapturedFrame(
2072 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2073 sink_.WaitForEncodedFrame(ntp_time);
2074 ntp_time += kFrameIntervalMs;
2075
2076 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2077 EXPECT_EQ(std::numeric_limits<int>::max(),
2078 video_source_.sink_wants().max_pixel_count);
2079 // Some framerate constraint should be set.
2080 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2081 EXPECT_LT(restricted_fps, kInputFps);
2082 video_source_.IncomingCapturedFrame(
2083 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2084 sink_.WaitForEncodedFrame(ntp_time);
2085 ntp_time += 100;
2086
Henrik Boström2671dac2020-05-19 16:29:09 +02002087 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002088 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2089 // Give the encoder queue time to process the change in degradation preference
2090 // by waiting for an encoded frame.
2091 video_source_.IncomingCapturedFrame(
2092 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2093 sink_.WaitForEncodedFrame(ntp_time);
2094 ntp_time += kFrameIntervalMs;
2095
2096 video_stream_encoder_->TriggerQualityLow();
2097 video_source_.IncomingCapturedFrame(
2098 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2099 sink_.WaitForEncodedFrame(ntp_time);
2100 ntp_time += kFrameIntervalMs;
2101
2102 // Some resolution constraint should be set.
2103 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2104 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2105 kFrameWidth * kFrameHeight);
2106 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2107
2108 int pixel_count = video_source_.sink_wants().max_pixel_count;
2109 // Triggering a CPU underuse should not change the sink wants since it has
2110 // not been overused for resolution since we changed degradation preference.
2111 video_stream_encoder_->TriggerCpuUnderuse();
2112 video_source_.IncomingCapturedFrame(
2113 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2114 sink_.WaitForEncodedFrame(ntp_time);
2115 ntp_time += kFrameIntervalMs;
2116 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2117 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2118
Evan Shrubsole64469032020-06-11 10:45:29 +02002119 // Change the degradation preference back. CPU underuse should not adapt since
2120 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002121 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002122 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2123 video_source_.IncomingCapturedFrame(
2124 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2125 sink_.WaitForEncodedFrame(ntp_time);
2126 ntp_time += 100;
2127 // Resolution adaptations is gone after changing degradation preference.
2128 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2129 EXPECT_EQ(std::numeric_limits<int>::max(),
2130 video_source_.sink_wants().max_pixel_count);
2131 // The fps adaptation from above is now back.
2132 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2133
2134 // Trigger CPU underuse.
2135 video_stream_encoder_->TriggerCpuUnderuse();
2136 video_source_.IncomingCapturedFrame(
2137 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2138 sink_.WaitForEncodedFrame(ntp_time);
2139 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002140 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2141
2142 // Trigger QP underuse, fps should return to normal.
2143 video_stream_encoder_->TriggerQualityHigh();
2144 video_source_.IncomingCapturedFrame(
2145 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2146 sink_.WaitForEncodedFrame(ntp_time);
2147 ntp_time += kFrameIntervalMs;
2148 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002149
2150 video_stream_encoder_->Stop();
2151}
2152
mflodmancc3d4422017-08-03 08:27:51 -07002153TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002155 DataRate::BitsPerSec(kTargetBitrateBps),
2156 DataRate::BitsPerSec(kTargetBitrateBps),
2157 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002158 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002159
sprangc5d62e22017-04-02 23:53:04 -07002160 const int kFrameWidth = 1280;
2161 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002162
Åsa Persson8c1bf952018-09-13 10:42:19 +02002163 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002164
kthelgason5e13d412016-12-01 03:59:51 -08002165 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002166 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002167 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002168 frame_timestamp += kFrameIntervalMs;
2169
perkj803d97f2016-11-01 11:45:46 -07002170 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002171 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002172 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002173 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002174 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002175 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002176
asapersson0944a802017-04-07 00:57:58 -07002177 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002178 // wanted resolution.
2179 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2180 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2181 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002182 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002183
2184 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002185 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002186 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002187 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002188 // Give the encoder queue time to process the change in degradation preference
2189 // by waiting for an encoded frame.
2190 new_video_source.IncomingCapturedFrame(
2191 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2192 sink_.WaitForEncodedFrame(frame_timestamp);
2193 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002194 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002195 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002196
sprangc5d62e22017-04-02 23:53:04 -07002197 // Force an input frame rate to be available, or the adaptation call won't
2198 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002199 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002200 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002201 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002202 stats_proxy_->SetMockStats(stats);
2203
mflodmancc3d4422017-08-03 08:27:51 -07002204 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002205 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002206 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002207 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002208 frame_timestamp += kFrameIntervalMs;
2209
2210 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002211 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002212 EXPECT_EQ(std::numeric_limits<int>::max(),
2213 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002214 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002215
asapersson02465b82017-04-10 01:12:52 -07002216 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002217 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2218 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002219 // Give the encoder queue time to process the change in degradation preference
2220 // by waiting for an encoded frame.
2221 new_video_source.IncomingCapturedFrame(
2222 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2223 sink_.WaitForEncodedFrame(frame_timestamp);
2224 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002225 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002226
mflodmancc3d4422017-08-03 08:27:51 -07002227 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002228 new_video_source.IncomingCapturedFrame(
2229 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002230 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002231 frame_timestamp += kFrameIntervalMs;
2232
2233 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002234 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002235
2236 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002237 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002238 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002239 // Give the encoder queue time to process the change in degradation preference
2240 // by waiting for an encoded frame.
2241 new_video_source.IncomingCapturedFrame(
2242 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2243 sink_.WaitForEncodedFrame(frame_timestamp);
2244 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002245 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2246 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002247 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002248 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002249
2250 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002251 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002252 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002253 // Give the encoder queue time to process the change in degradation preference
2254 // by waiting for an encoded frame.
2255 new_video_source.IncomingCapturedFrame(
2256 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2257 sink_.WaitForEncodedFrame(frame_timestamp);
2258 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002259 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2260 EXPECT_EQ(std::numeric_limits<int>::max(),
2261 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002262 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002263
mflodmancc3d4422017-08-03 08:27:51 -07002264 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002265}
2266
mflodmancc3d4422017-08-03 08:27:51 -07002267TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002268 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002269 DataRate::BitsPerSec(kTargetBitrateBps),
2270 DataRate::BitsPerSec(kTargetBitrateBps),
2271 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002272
asaperssonfab67072017-04-04 05:51:49 -07002273 const int kWidth = 1280;
2274 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002275 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002276 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002277 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2278 EXPECT_FALSE(stats.bw_limited_resolution);
2279 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2280
2281 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002282 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002283 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002284 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002285
2286 stats = stats_proxy_->GetStats();
2287 EXPECT_TRUE(stats.bw_limited_resolution);
2288 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2289
2290 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002291 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002292 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002293 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002294
2295 stats = stats_proxy_->GetStats();
2296 EXPECT_FALSE(stats.bw_limited_resolution);
2297 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2298 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2299
mflodmancc3d4422017-08-03 08:27:51 -07002300 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002301}
2302
mflodmancc3d4422017-08-03 08:27:51 -07002303TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002305 DataRate::BitsPerSec(kTargetBitrateBps),
2306 DataRate::BitsPerSec(kTargetBitrateBps),
2307 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002308
2309 const int kWidth = 1280;
2310 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002311 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002312 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002313 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2314 EXPECT_FALSE(stats.cpu_limited_resolution);
2315 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2316
2317 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002318 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002319 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002320 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002321
2322 stats = stats_proxy_->GetStats();
2323 EXPECT_TRUE(stats.cpu_limited_resolution);
2324 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2325
2326 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002327 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002328 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002329 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002330
2331 stats = stats_proxy_->GetStats();
2332 EXPECT_FALSE(stats.cpu_limited_resolution);
2333 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002334 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002335
mflodmancc3d4422017-08-03 08:27:51 -07002336 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002337}
2338
mflodmancc3d4422017-08-03 08:27:51 -07002339TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002340 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002341 DataRate::BitsPerSec(kTargetBitrateBps),
2342 DataRate::BitsPerSec(kTargetBitrateBps),
2343 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002344
asaperssonfab67072017-04-04 05:51:49 -07002345 const int kWidth = 1280;
2346 const int kHeight = 720;
2347 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002348 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002349 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002350 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002351 EXPECT_FALSE(stats.cpu_limited_resolution);
2352 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2353
asaperssonfab67072017-04-04 05:51:49 -07002354 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002355 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002356 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002357 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002358 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002359 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002360 EXPECT_TRUE(stats.cpu_limited_resolution);
2361 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2362
2363 // Set new source with adaptation still enabled.
2364 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002365 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002366 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002367
asaperssonfab67072017-04-04 05:51:49 -07002368 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002369 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002370 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002371 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002372 EXPECT_TRUE(stats.cpu_limited_resolution);
2373 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2374
2375 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002376 video_stream_encoder_->SetSource(&new_video_source,
2377 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002378
asaperssonfab67072017-04-04 05:51:49 -07002379 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002380 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002381 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002382 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002383 EXPECT_FALSE(stats.cpu_limited_resolution);
2384 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2385
2386 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002387 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002388 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002389
asaperssonfab67072017-04-04 05:51:49 -07002390 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002391 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002392 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002393 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002394 EXPECT_TRUE(stats.cpu_limited_resolution);
2395 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2396
asaperssonfab67072017-04-04 05:51:49 -07002397 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002398 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002399 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002400 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002401 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002402 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002403 EXPECT_FALSE(stats.cpu_limited_resolution);
2404 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002405 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002406
mflodmancc3d4422017-08-03 08:27:51 -07002407 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002408}
2409
mflodmancc3d4422017-08-03 08:27:51 -07002410TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002411 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002412 DataRate::BitsPerSec(kTargetBitrateBps),
2413 DataRate::BitsPerSec(kTargetBitrateBps),
2414 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002415
asaperssonfab67072017-04-04 05:51:49 -07002416 const int kWidth = 1280;
2417 const int kHeight = 720;
2418 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002419 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002420 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002421 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002422 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002423 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002424
2425 // Set new source with adaptation still enabled.
2426 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002427 video_stream_encoder_->SetSource(&new_video_source,
2428 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002429
asaperssonfab67072017-04-04 05:51:49 -07002430 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002431 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002432 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002433 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002434 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002435 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002436
asaperssonfab67072017-04-04 05:51:49 -07002437 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002438 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002439 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002440 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002441 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002442 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002443 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002444 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002445
asaperssonfab67072017-04-04 05:51:49 -07002446 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002447 video_stream_encoder_->SetSource(&new_video_source,
2448 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002449
asaperssonfab67072017-04-04 05:51:49 -07002450 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002451 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002452 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002453 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002454 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002455 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002456
asapersson02465b82017-04-10 01:12:52 -07002457 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002458 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002459 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002460
asaperssonfab67072017-04-04 05:51:49 -07002461 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002462 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002463 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002464 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002465 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002466 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2467 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002468
mflodmancc3d4422017-08-03 08:27:51 -07002469 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002470}
2471
mflodmancc3d4422017-08-03 08:27:51 -07002472TEST_F(VideoStreamEncoderTest,
2473 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002475 DataRate::BitsPerSec(kTargetBitrateBps),
2476 DataRate::BitsPerSec(kTargetBitrateBps),
2477 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002478
2479 const int kWidth = 1280;
2480 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002481 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002482 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002483 video_source_.IncomingCapturedFrame(
2484 CreateFrame(timestamp_ms, kWidth, kHeight));
2485 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002486 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2488 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2489
2490 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002491 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002492 timestamp_ms += kFrameIntervalMs;
2493 video_source_.IncomingCapturedFrame(
2494 CreateFrame(timestamp_ms, kWidth, kHeight));
2495 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002496 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2497 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2498 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2499
2500 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002501 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002502 timestamp_ms += kFrameIntervalMs;
2503 video_source_.IncomingCapturedFrame(
2504 CreateFrame(timestamp_ms, kWidth, kHeight));
2505 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002506 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2507 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2508 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2509
Niels Möller4db138e2018-04-19 09:04:13 +02002510 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002511 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002512
2513 VideoEncoderConfig video_encoder_config;
2514 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2515 // Make format different, to force recreation of encoder.
2516 video_encoder_config.video_format.parameters["foo"] = "foo";
2517 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002518 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002519 timestamp_ms += kFrameIntervalMs;
2520 video_source_.IncomingCapturedFrame(
2521 CreateFrame(timestamp_ms, kWidth, kHeight));
2522 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002523 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2525 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2526
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002528}
2529
mflodmancc3d4422017-08-03 08:27:51 -07002530TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002531 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002532 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002533 DataRate::BitsPerSec(kTargetBitrateBps),
2534 DataRate::BitsPerSec(kTargetBitrateBps),
2535 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2536
2537 const int kWidth = 1280;
2538 const int kHeight = 720;
2539 int sequence = 1;
2540
2541 // Enable BALANCED preference, no initial limitation.
2542 test::FrameForwarder source;
2543 video_stream_encoder_->SetSource(&source,
2544 webrtc::DegradationPreference::BALANCED);
2545 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2546 WaitForEncodedFrame(sequence++);
2547 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2548 EXPECT_FALSE(stats.cpu_limited_resolution);
2549 EXPECT_FALSE(stats.cpu_limited_framerate);
2550 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2551
2552 // Trigger CPU overuse, should now adapt down.
2553 video_stream_encoder_->TriggerCpuOveruse();
2554 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2555 WaitForEncodedFrame(sequence++);
2556 stats = stats_proxy_->GetStats();
2557 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2558
2559 // Set new degradation preference should clear restrictions since we changed
2560 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002561 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002562 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2563 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2564 WaitForEncodedFrame(sequence++);
2565 stats = stats_proxy_->GetStats();
2566 EXPECT_FALSE(stats.cpu_limited_resolution);
2567 EXPECT_FALSE(stats.cpu_limited_framerate);
2568 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2569
2570 // Force an input frame rate to be available, or the adaptation call won't
2571 // know what framerate to adapt from.
2572 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2573 mock_stats.input_frame_rate = 30;
2574 stats_proxy_->SetMockStats(mock_stats);
2575 video_stream_encoder_->TriggerCpuOveruse();
2576 stats_proxy_->ResetMockStats();
2577 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2578 WaitForEncodedFrame(sequence++);
2579
2580 // We have now adapted once.
2581 stats = stats_proxy_->GetStats();
2582 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2583
2584 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002585 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2586 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002587 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2588 WaitForEncodedFrame(sequence++);
2589 stats = stats_proxy_->GetStats();
2590 EXPECT_FALSE(stats.cpu_limited_resolution);
2591 EXPECT_FALSE(stats.cpu_limited_framerate);
2592 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2593
2594 video_stream_encoder_->Stop();
2595}
2596
2597TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002598 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002600 DataRate::BitsPerSec(kTargetBitrateBps),
2601 DataRate::BitsPerSec(kTargetBitrateBps),
2602 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002603
asapersson0944a802017-04-07 00:57:58 -07002604 const int kWidth = 1280;
2605 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002606 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002607
asaperssonfab67072017-04-04 05:51:49 -07002608 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002609 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002610 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002611 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002612 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002613 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2614
asapersson02465b82017-04-10 01:12:52 -07002615 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002616 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002617 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002618 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002619 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002620 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002621 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002622 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2623
2624 // Set new source with adaptation still enabled.
2625 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002626 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002627 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002628
2629 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002630 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002631 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002632 stats = stats_proxy_->GetStats();
2633 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002634 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002635 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2636
sprangc5d62e22017-04-02 23:53:04 -07002637 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002638 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002639 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002640 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002641 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002642 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002643 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002644 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002645 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002646 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002647 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2648
sprangc5d62e22017-04-02 23:53:04 -07002649 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002650 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002651 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2652 mock_stats.input_frame_rate = 30;
2653 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002654 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002655 stats_proxy_->ResetMockStats();
2656
2657 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002658 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002659 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002660
2661 // Framerate now adapted.
2662 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002663 EXPECT_FALSE(stats.cpu_limited_resolution);
2664 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002665 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2666
2667 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002668 video_stream_encoder_->SetSource(&new_video_source,
2669 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002670 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002671 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002672 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002673
2674 stats = stats_proxy_->GetStats();
2675 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002676 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002677 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2678
2679 // Try to trigger overuse. Should not succeed.
2680 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002681 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002682 stats_proxy_->ResetMockStats();
2683
2684 stats = stats_proxy_->GetStats();
2685 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002686 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002687 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2688
2689 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002690 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002691 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002692 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002693 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002694 stats = stats_proxy_->GetStats();
2695 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002696 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002697 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002698
2699 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002700 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002701 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002702 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002703 stats = stats_proxy_->GetStats();
2704 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002705 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002706 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2707
2708 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002709 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002710 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002711 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002712 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002713 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002714 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002715 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002716 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002717 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002718 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2719
2720 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002721 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002722 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002723 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002724 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002725 stats = stats_proxy_->GetStats();
2726 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002727 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002728 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002729 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002730
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002732}
2733
mflodmancc3d4422017-08-03 08:27:51 -07002734TEST_F(VideoStreamEncoderTest,
2735 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002736 const int kWidth = 1280;
2737 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002739 DataRate::BitsPerSec(kTargetBitrateBps),
2740 DataRate::BitsPerSec(kTargetBitrateBps),
2741 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002742
asaperssonfab67072017-04-04 05:51:49 -07002743 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002744 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002745
asaperssonfab67072017-04-04 05:51:49 -07002746 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002747 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002748
asaperssonfab67072017-04-04 05:51:49 -07002749 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002750 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002751
asaperssonfab67072017-04-04 05:51:49 -07002752 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002753 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002754
kthelgason876222f2016-11-29 01:44:11 -08002755 // Expect a scale down.
2756 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002757 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002758
asapersson02465b82017-04-10 01:12:52 -07002759 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002760 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002761 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002762 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002763
asaperssonfab67072017-04-04 05:51:49 -07002764 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002765 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002766 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002767 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002768
asaperssonfab67072017-04-04 05:51:49 -07002769 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002770 EXPECT_EQ(std::numeric_limits<int>::max(),
2771 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002772
asaperssonfab67072017-04-04 05:51:49 -07002773 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002774 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002775 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002776 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002777
asapersson02465b82017-04-10 01:12:52 -07002778 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002779 EXPECT_EQ(std::numeric_limits<int>::max(),
2780 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002781
mflodmancc3d4422017-08-03 08:27:51 -07002782 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002783}
2784
mflodmancc3d4422017-08-03 08:27:51 -07002785TEST_F(VideoStreamEncoderTest,
2786 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002787 const int kWidth = 1280;
2788 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002790 DataRate::BitsPerSec(kTargetBitrateBps),
2791 DataRate::BitsPerSec(kTargetBitrateBps),
2792 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002793
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002794 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002795 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002796 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002797 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002798
2799 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002800 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002801 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2803 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2804
2805 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002807 EXPECT_THAT(source.sink_wants(),
2808 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002809 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2810 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2811 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2812
2813 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002814 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002815 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2816 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2817 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2818
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002820}
2821
mflodmancc3d4422017-08-03 08:27:51 -07002822TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002823 const int kWidth = 1280;
2824 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002826 DataRate::BitsPerSec(kTargetBitrateBps),
2827 DataRate::BitsPerSec(kTargetBitrateBps),
2828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002829
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002830 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002831 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002832 video_stream_encoder_->SetSource(&source,
2833 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002834 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2835 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002836 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002837
2838 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002839 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002840 EXPECT_THAT(source.sink_wants(),
2841 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002842 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2843 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2844 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2845
2846 // Trigger adapt down for same input resolution, expect no change.
2847 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2848 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002849 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002850 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2851 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2852 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2853
2854 // Trigger adapt down for larger input resolution, expect no change.
2855 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2856 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002858 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2859 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2861
mflodmancc3d4422017-08-03 08:27:51 -07002862 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002863}
2864
mflodmancc3d4422017-08-03 08:27:51 -07002865TEST_F(VideoStreamEncoderTest,
2866 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002867 const int kWidth = 1280;
2868 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002870 DataRate::BitsPerSec(kTargetBitrateBps),
2871 DataRate::BitsPerSec(kTargetBitrateBps),
2872 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002873
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002874 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002875 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002876 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002877 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002878
2879 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002880 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002881 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002882 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2883 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2884
2885 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002886 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002887 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002888 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2889 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2890
mflodmancc3d4422017-08-03 08:27:51 -07002891 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002892}
2893
mflodmancc3d4422017-08-03 08:27:51 -07002894TEST_F(VideoStreamEncoderTest,
2895 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002896 const int kWidth = 1280;
2897 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002899 DataRate::BitsPerSec(kTargetBitrateBps),
2900 DataRate::BitsPerSec(kTargetBitrateBps),
2901 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002902
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002903 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002904 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002905 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002906 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002907
2908 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002909 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002910 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002911 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002912 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2913
2914 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002915 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002916 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002917 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002918 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2919
mflodmancc3d4422017-08-03 08:27:51 -07002920 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002921}
2922
mflodmancc3d4422017-08-03 08:27:51 -07002923TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002924 const int kWidth = 1280;
2925 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002926 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002927 DataRate::BitsPerSec(kTargetBitrateBps),
2928 DataRate::BitsPerSec(kTargetBitrateBps),
2929 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002930
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002931 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002932 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002933 video_stream_encoder_->SetSource(&source,
2934 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002935
2936 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2937 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002938 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002939 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2940 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2941 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2942
2943 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002944 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002945 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002946 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2947 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2948 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2949
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002951}
2952
mflodmancc3d4422017-08-03 08:27:51 -07002953TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002954 const int kWidth = 1280;
2955 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002957 DataRate::BitsPerSec(kTargetBitrateBps),
2958 DataRate::BitsPerSec(kTargetBitrateBps),
2959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002960
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002961 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002962 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002963 video_stream_encoder_->SetSource(&source,
2964 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002965
2966 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2967 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002968 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002969 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2970 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2971 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2972
2973 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002974 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002975 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002976 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2977 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2978 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2979
mflodmancc3d4422017-08-03 08:27:51 -07002980 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002981}
2982
mflodmancc3d4422017-08-03 08:27:51 -07002983TEST_F(VideoStreamEncoderTest,
2984 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002985 const int kWidth = 1280;
2986 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002988 DataRate::BitsPerSec(kTargetBitrateBps),
2989 DataRate::BitsPerSec(kTargetBitrateBps),
2990 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002991
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002992 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02002993 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07002994 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002995 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002996 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002997
2998 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002999 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003000 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3002 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3003
3004 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003005 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003006 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003007 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003008 EXPECT_THAT(source.sink_wants(),
3009 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003010 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3011 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3012
3013 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003014 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003015 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003016 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3017 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3018 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3019
mflodmancc3d4422017-08-03 08:27:51 -07003020 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003021}
3022
mflodmancc3d4422017-08-03 08:27:51 -07003023TEST_F(VideoStreamEncoderTest,
3024 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003025 const int kWidth = 1280;
3026 const int kHeight = 720;
3027 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003028 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003029 DataRate::BitsPerSec(kTargetBitrateBps),
3030 DataRate::BitsPerSec(kTargetBitrateBps),
3031 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003032
3033 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3034 stats.input_frame_rate = kInputFps;
3035 stats_proxy_->SetMockStats(stats);
3036
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003037 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003038 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3039 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003040 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003041
3042 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003043 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003044 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3045 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003046 EXPECT_THAT(video_source_.sink_wants(),
3047 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003048
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003049 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003050 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003051 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003052 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003053 // Give the encoder queue time to process the change in degradation preference
3054 // by waiting for an encoded frame.
3055 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3056 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003057 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003058
3059 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003060 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003061 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3062 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003063 EXPECT_THAT(new_video_source.sink_wants(),
3064 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003065
3066 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003067 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003068 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003069
mflodmancc3d4422017-08-03 08:27:51 -07003070 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003071}
3072
mflodmancc3d4422017-08-03 08:27:51 -07003073TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003074 const int kWidth = 1280;
3075 const int kHeight = 720;
3076 const size_t kNumFrames = 10;
3077
Henrik Boström381d1092020-05-12 18:49:07 +02003078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003079 DataRate::BitsPerSec(kTargetBitrateBps),
3080 DataRate::BitsPerSec(kTargetBitrateBps),
3081 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003082
asaperssond0de2952017-04-21 01:47:31 -07003083 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003084 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003085 video_source_.set_adaptation_enabled(true);
3086
3087 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3088 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3089
3090 int downscales = 0;
3091 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003092 video_source_.IncomingCapturedFrame(
3093 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3094 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003095
asaperssonfab67072017-04-04 05:51:49 -07003096 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003097 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003098 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003099 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003100
3101 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3102 ++downscales;
3103
3104 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3105 EXPECT_EQ(downscales,
3106 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3107 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003108 }
mflodmancc3d4422017-08-03 08:27:51 -07003109 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003110}
3111
mflodmancc3d4422017-08-03 08:27:51 -07003112TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003113 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3114 const int kWidth = 1280;
3115 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003117 DataRate::BitsPerSec(kTargetBitrateBps),
3118 DataRate::BitsPerSec(kTargetBitrateBps),
3119 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003120
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003121 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003122 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003123 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003124 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003125 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003126
Åsa Persson8c1bf952018-09-13 10:42:19 +02003127 int64_t timestamp_ms = kFrameIntervalMs;
3128 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003129 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003130 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003131 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3132 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3133
3134 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003135 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003136 timestamp_ms += kFrameIntervalMs;
3137 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3138 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003139 EXPECT_THAT(source.sink_wants(),
3140 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003141 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3142 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3143
3144 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003145 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003146 timestamp_ms += kFrameIntervalMs;
3147 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003148 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003149 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003150 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3151 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3152
3153 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003154 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003155 timestamp_ms += kFrameIntervalMs;
3156 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3157 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003158 EXPECT_THAT(source.sink_wants(),
3159 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003160 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3161 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3162
3163 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003164 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003165 timestamp_ms += kFrameIntervalMs;
3166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003167 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003168 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003169 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3170 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3171
mflodmancc3d4422017-08-03 08:27:51 -07003172 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003173}
3174
mflodmancc3d4422017-08-03 08:27:51 -07003175TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003176 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3177 const int kWidth = 1280;
3178 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003180 DataRate::BitsPerSec(kTargetBitrateBps),
3181 DataRate::BitsPerSec(kTargetBitrateBps),
3182 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003183
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003184 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003185 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003186 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003187 video_stream_encoder_->SetSource(&source,
3188 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003189
Åsa Persson8c1bf952018-09-13 10:42:19 +02003190 int64_t timestamp_ms = kFrameIntervalMs;
3191 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003192 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003193 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3195 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3196
3197 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003198 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003199 timestamp_ms += kFrameIntervalMs;
3200 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3201 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003202 EXPECT_THAT(source.sink_wants(),
3203 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003204 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3205 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3206
3207 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003208 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003209 timestamp_ms += kFrameIntervalMs;
3210 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003211 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003212 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003213 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3214 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3215
3216 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003217 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003218 timestamp_ms += kFrameIntervalMs;
3219 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3220 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003221 EXPECT_THAT(source.sink_wants(),
3222 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003223 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3224 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3225
3226 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003227 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003228 timestamp_ms += kFrameIntervalMs;
3229 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003230 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003231 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003232 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3233 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3234
mflodmancc3d4422017-08-03 08:27:51 -07003235 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003236}
3237
Sergey Silkin41c650b2019-10-14 13:12:19 +02003238TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3239 fake_encoder_.SetResolutionBitrateLimits(
3240 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3241
Henrik Boström381d1092020-05-12 18:49:07 +02003242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003243 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3244 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3245 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3246 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003247
3248 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003249 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003250 source.set_adaptation_enabled(true);
3251 video_stream_encoder_->SetSource(
3252 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3253
3254 // Insert 720p frame.
3255 int64_t timestamp_ms = kFrameIntervalMs;
3256 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3257 WaitForEncodedFrame(1280, 720);
3258
3259 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003261 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3262 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3263 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3264 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003265 video_stream_encoder_->TriggerQualityLow();
3266
3267 // Insert 720p frame. It should be downscaled and encoded.
3268 timestamp_ms += kFrameIntervalMs;
3269 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3270 WaitForEncodedFrame(960, 540);
3271
3272 // Trigger adapt up. Higher resolution should not be requested duo to lack
3273 // of bitrate.
3274 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003275 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003276
3277 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003279 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3280 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3281 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3282 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003283
3284 // Trigger adapt up. Higher resolution should be requested.
3285 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003286 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003287
3288 video_stream_encoder_->Stop();
3289}
3290
3291TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3292 fake_encoder_.SetResolutionBitrateLimits(
3293 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3294
3295 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003297 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3298 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3299 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3300 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003301
3302 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003303 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003304 source.set_adaptation_enabled(true);
3305 video_stream_encoder_->SetSource(
3306 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3307
3308 // Insert 720p frame. It should be dropped and lower resolution should be
3309 // requested.
3310 int64_t timestamp_ms = kFrameIntervalMs;
3311 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3312 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003313 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003314
3315 // Insert 720p frame. It should be downscaled and encoded.
3316 timestamp_ms += kFrameIntervalMs;
3317 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3318 WaitForEncodedFrame(960, 540);
3319
3320 video_stream_encoder_->Stop();
3321}
3322
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003323class BalancedDegradationTest : public VideoStreamEncoderTest {
3324 protected:
3325 void SetupTest() {
3326 // Reset encoder for field trials to take effect.
3327 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003328 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003329
3330 // Enable BALANCED preference.
3331 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003332 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3333 }
3334
3335 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003337 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3338 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003339 }
3340
Åsa Persson45b176f2019-09-30 11:19:05 +02003341 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003342 timestamp_ms_ += kFrameIntervalMs;
3343 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003344 }
3345
3346 void InsertFrameAndWaitForEncoded() {
3347 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003348 sink_.WaitForEncodedFrame(timestamp_ms_);
3349 }
3350
3351 const int kWidth = 640; // pixels:640x360=230400
3352 const int kHeight = 360;
3353 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3354 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003355 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003356};
3357
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003358TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003359 test::ScopedFieldTrials field_trials(
3360 "WebRTC-Video-BalancedDegradationSettings/"
3361 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3362 SetupTest();
3363
3364 // Force input frame rate.
3365 const int kInputFps = 24;
3366 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3367 stats.input_frame_rate = kInputFps;
3368 stats_proxy_->SetMockStats(stats);
3369
Åsa Persson45b176f2019-09-30 11:19:05 +02003370 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003371 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003372
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003373 // Trigger adapt down, expect scaled down framerate and resolution,
3374 // since Fps diff (input-requested:0) < threshold.
3375 video_stream_encoder_->TriggerQualityLow();
3376 EXPECT_THAT(source_.sink_wants(),
3377 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003378
3379 video_stream_encoder_->Stop();
3380}
3381
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003382TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003383 test::ScopedFieldTrials field_trials(
3384 "WebRTC-Video-BalancedDegradationSettings/"
3385 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3386 SetupTest();
3387
3388 // Force input frame rate.
3389 const int kInputFps = 25;
3390 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3391 stats.input_frame_rate = kInputFps;
3392 stats_proxy_->SetMockStats(stats);
3393
Åsa Persson45b176f2019-09-30 11:19:05 +02003394 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003395 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003396
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003397 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3398 // Fps diff (input-requested:1) == threshold.
3399 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003400 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003401
3402 video_stream_encoder_->Stop();
3403}
3404
3405TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3406 test::ScopedFieldTrials field_trials(
3407 "WebRTC-Video-BalancedDegradationSettings/"
3408 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3409 SetupTest();
3410
3411 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3412
Åsa Persson45b176f2019-09-30 11:19:05 +02003413 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003414 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003415
3416 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3417 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003418 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003419
3420 video_stream_encoder_->Stop();
3421}
3422
Åsa Perssonccfb3402019-09-25 15:13:04 +02003423TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003424 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003425 "WebRTC-Video-BalancedDegradationSettings/"
3426 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003427 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003428
Åsa Persson1b247f12019-08-14 17:26:39 +02003429 const int kMinBitrateBps = 425000;
3430 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003431 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003432
Åsa Persson45b176f2019-09-30 11:19:05 +02003433 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003434 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003435 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
3437 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3438 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003439 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003440 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003441 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3442
3443 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3444 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003445 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003446 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003447 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3448
Åsa Persson30ab0152019-08-27 12:22:33 +02003449 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3450 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003451 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003452 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003453 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003454 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3455
3456 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003457 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003458 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003459 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003460
Åsa Persson30ab0152019-08-27 12:22:33 +02003461 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003462 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003463 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003464 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003465 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003466 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3467
3468 video_stream_encoder_->Stop();
3469}
3470
Åsa Perssonccfb3402019-09-25 15:13:04 +02003471TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003472 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3473 test::ScopedFieldTrials field_trials(
3474 "WebRTC-Video-BalancedDegradationSettings/"
3475 "pixels:57600|129600|230400,fps:7|24|24/");
3476 SetupTest();
3477 OnBitrateUpdated(kLowTargetBitrateBps);
3478
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003479 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003480
3481 // Insert frame, expect scaled down:
3482 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3483 InsertFrame();
3484 EXPECT_FALSE(WaitForFrame(1000));
3485 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3486 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3487
3488 // Insert frame, expect scaled down:
3489 // resolution (320x180@24fps).
3490 InsertFrame();
3491 EXPECT_FALSE(WaitForFrame(1000));
3492 EXPECT_LT(source_.sink_wants().max_pixel_count,
3493 source_.last_wants().max_pixel_count);
3494 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3495
3496 // Frame should not be dropped (min pixels per frame reached).
3497 InsertFrameAndWaitForEncoded();
3498
3499 video_stream_encoder_->Stop();
3500}
3501
3502TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003503 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003504 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003505 "WebRTC-Video-BalancedDegradationSettings/"
3506 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003507 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003508
Åsa Persson30ab0152019-08-27 12:22:33 +02003509 const int kResolutionMinBitrateBps = 435000;
3510 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003511 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003512
Åsa Persson45b176f2019-09-30 11:19:05 +02003513 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003514 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003515 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3516
3517 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3518 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003519 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003520 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003521 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3522
3523 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3524 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003525 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003526 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003527 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3528
3529 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3530 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003531 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003532 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003533 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3534
Åsa Persson30ab0152019-08-27 12:22:33 +02003535 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3536 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003537 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003538 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003539 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3540
3541 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3542 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003543 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003544 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3545
3546 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003547 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003548 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003549 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003550 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003551 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3552
3553 video_stream_encoder_->Stop();
3554}
3555
Åsa Perssonccfb3402019-09-25 15:13:04 +02003556TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003557 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003558 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003559 "WebRTC-Video-BalancedDegradationSettings/"
3560 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003561 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003562
Åsa Persson30ab0152019-08-27 12:22:33 +02003563 const int kMinBitrateBps = 425000;
3564 const int kTooLowMinBitrateBps = 424000;
3565 const int kResolutionMinBitrateBps = 435000;
3566 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003567 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003568
Åsa Persson45b176f2019-09-30 11:19:05 +02003569 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003570 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003571 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3572
3573 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3574 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003575 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003576 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003577 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3578
3579 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3580 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003581 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003582 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003583 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3584
3585 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3586 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003587 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003588 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003589 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3590
3591 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3592 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003593 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003594 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3595
3596 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003597 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003598 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003599 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003600 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003601 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3602
3603 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003604 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003605 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003606 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003607 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3608
3609 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003610 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003611 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003612 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003613 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003614 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615
Åsa Persson1b247f12019-08-14 17:26:39 +02003616 video_stream_encoder_->Stop();
3617}
3618
mflodmancc3d4422017-08-03 08:27:51 -07003619TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003620 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3621 const int kWidth = 1280;
3622 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003624 DataRate::BitsPerSec(kTargetBitrateBps),
3625 DataRate::BitsPerSec(kTargetBitrateBps),
3626 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003627
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003628 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003629 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003630 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003631 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003632 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003633
Åsa Persson8c1bf952018-09-13 10:42:19 +02003634 int64_t timestamp_ms = kFrameIntervalMs;
3635 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003636 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003637 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3639 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3640 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3641 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3642
3643 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003644 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003645 timestamp_ms += kFrameIntervalMs;
3646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3647 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003648 EXPECT_THAT(source.sink_wants(),
3649 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003650 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3652 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3654
3655 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003656 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003657 timestamp_ms += kFrameIntervalMs;
3658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3659 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003660 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003661 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3663 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3664 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3665
Jonathan Yubc771b72017-12-08 17:04:29 -08003666 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003667 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003668 timestamp_ms += kFrameIntervalMs;
3669 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3670 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003671 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003672 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003674 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003675 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3676
Jonathan Yubc771b72017-12-08 17:04:29 -08003677 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003678 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003679 timestamp_ms += kFrameIntervalMs;
3680 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3681 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003682 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003683 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003684 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3685 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3686 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3687 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3688
Jonathan Yubc771b72017-12-08 17:04:29 -08003689 // Trigger quality adapt down, expect no change (min resolution reached).
3690 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003691 timestamp_ms += kFrameIntervalMs;
3692 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3693 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003694 EXPECT_THAT(source.sink_wants(), FpsMax());
3695 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003696 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3697 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3698 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3699 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3700
Evan Shrubsole64469032020-06-11 10:45:29 +02003701 // Trigger quality adapt up, expect upscaled resolution (480x270).
3702 video_stream_encoder_->TriggerQualityHigh();
3703 timestamp_ms += kFrameIntervalMs;
3704 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3705 WaitForEncodedFrame(timestamp_ms);
3706 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3707 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3708 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3709 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3710 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3711
3712 // Trigger quality and cpu adapt up since both are most limited, expect
3713 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003714 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003715 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003716 timestamp_ms += kFrameIntervalMs;
3717 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3718 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003719 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003720 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3721 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3722 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003723 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003724
Evan Shrubsole64469032020-06-11 10:45:29 +02003725 // Trigger quality and cpu adapt up since both are most limited, expect
3726 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003727 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003728 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003729 timestamp_ms += kFrameIntervalMs;
3730 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3731 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003732 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003733 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003734 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003735 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003736 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3737 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003738
Evan Shrubsole64469032020-06-11 10:45:29 +02003739 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3740 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003741 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003742 timestamp_ms += kFrameIntervalMs;
3743 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3744 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003745 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003746 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3747 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003748 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003749 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003750
3751 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003752 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003753 timestamp_ms += kFrameIntervalMs;
3754 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003755 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003756 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003758 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3759 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003760 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003761 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003762
mflodmancc3d4422017-08-03 08:27:51 -07003763 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003764}
3765
mflodmancc3d4422017-08-03 08:27:51 -07003766TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003767 const int kWidth = 640;
3768 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003769
Henrik Boström381d1092020-05-12 18:49:07 +02003770 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003771 DataRate::BitsPerSec(kTargetBitrateBps),
3772 DataRate::BitsPerSec(kTargetBitrateBps),
3773 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003774
perkj803d97f2016-11-01 11:45:46 -07003775 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003776 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003777 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003778 }
3779
mflodmancc3d4422017-08-03 08:27:51 -07003780 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003781 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003782 video_source_.IncomingCapturedFrame(CreateFrame(
3783 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003784 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003785 }
3786
mflodmancc3d4422017-08-03 08:27:51 -07003787 video_stream_encoder_->Stop();
3788 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003789 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003790
Ying Wangef3998f2019-12-09 13:06:53 +01003791 EXPECT_METRIC_EQ(
3792 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3793 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003794 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3795}
3796
mflodmancc3d4422017-08-03 08:27:51 -07003797TEST_F(VideoStreamEncoderTest,
3798 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003799 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003800 DataRate::BitsPerSec(kTargetBitrateBps),
3801 DataRate::BitsPerSec(kTargetBitrateBps),
3802 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003803 const int kWidth = 640;
3804 const int kHeight = 360;
3805
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003806 video_stream_encoder_->SetSource(&video_source_,
3807 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003808
3809 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3810 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003811 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003812 }
3813
mflodmancc3d4422017-08-03 08:27:51 -07003814 video_stream_encoder_->Stop();
3815 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003816 stats_proxy_.reset();
3817
3818 EXPECT_EQ(0,
3819 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3820}
3821
mflodmancc3d4422017-08-03 08:27:51 -07003822TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003823 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003824 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003825
3826 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003827 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003828 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003829 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3830 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003831
sprang57c2fff2017-01-16 06:24:02 -08003832 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003833 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003835 DataRate::BitsPerSec(kLowTargetBitrateBps),
3836 DataRate::BitsPerSec(kLowTargetBitrateBps),
3837 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003838
sprang57c2fff2017-01-16 06:24:02 -08003839 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003840 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
3841 WaitForEncodedFrame(CurrentTimeMs());
Erik Språng5056af02019-09-02 15:53:11 +02003842 VideoBitrateAllocation bitrate_allocation =
3843 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003844 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003845 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003846 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003847
3848 // Not called on second frame.
3849 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3850 .Times(0);
3851 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003852 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
3853 WaitForEncodedFrame(CurrentTimeMs());
3854 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003855
3856 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003857 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3858 .Times(1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003859 const int64_t start_time_ms = CurrentTimeMs();
3860 while (CurrentTimeMs() - start_time_ms < kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01003861 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003862 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
3863 WaitForEncodedFrame(CurrentTimeMs());
3864 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003865 }
3866
mflodmancc3d4422017-08-03 08:27:51 -07003867 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003868}
3869
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003870TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3871 // 2 TLs configured, temporal layers supported by encoder.
3872 const int kNumTemporalLayers = 2;
3873 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3874 fake_encoder_.SetTemporalLayersSupported(0, true);
3875
3876 // Bitrate allocated across temporal layers.
3877 const int kTl0Bps = kTargetBitrateBps *
3878 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003879 kNumTemporalLayers, /*temporal_id*/ 0,
3880 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003881 const int kTl1Bps = kTargetBitrateBps *
3882 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003883 kNumTemporalLayers, /*temporal_id*/ 1,
3884 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003885 VideoBitrateAllocation expected_bitrate;
3886 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3887 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3888
3889 VerifyAllocatedBitrate(expected_bitrate);
3890 video_stream_encoder_->Stop();
3891}
3892
3893TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3894 // 2 TLs configured, temporal layers not supported by encoder.
3895 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3896 fake_encoder_.SetTemporalLayersSupported(0, false);
3897
3898 // Temporal layers not supported by the encoder.
3899 // Total bitrate should be at ti:0.
3900 VideoBitrateAllocation expected_bitrate;
3901 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3902
3903 VerifyAllocatedBitrate(expected_bitrate);
3904 video_stream_encoder_->Stop();
3905}
3906
3907TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3908 // 2 TLs configured, temporal layers only supported for first stream.
3909 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3910 fake_encoder_.SetTemporalLayersSupported(0, true);
3911 fake_encoder_.SetTemporalLayersSupported(1, false);
3912
3913 const int kS0Bps = 150000;
3914 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003915 kS0Bps *
3916 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3917 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003918 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003919 kS0Bps *
3920 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3921 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003922 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3923 // Temporal layers not supported by si:1.
3924 VideoBitrateAllocation expected_bitrate;
3925 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3926 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3927 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3928
3929 VerifyAllocatedBitrate(expected_bitrate);
3930 video_stream_encoder_->Stop();
3931}
3932
Niels Möller7dc26b72017-12-06 10:27:48 +01003933TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3934 const int kFrameWidth = 1280;
3935 const int kFrameHeight = 720;
3936 const int kFramerate = 24;
3937
Henrik Boström381d1092020-05-12 18:49:07 +02003938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003939 DataRate::BitsPerSec(kTargetBitrateBps),
3940 DataRate::BitsPerSec(kTargetBitrateBps),
3941 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003942 test::FrameForwarder source;
3943 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003944 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003945
3946 // Insert a single frame, triggering initial configuration.
3947 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3948 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3949
3950 EXPECT_EQ(
3951 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3952 kDefaultFramerate);
3953
3954 // Trigger reconfigure encoder (without resetting the entire instance).
3955 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003956 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003957 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3958 video_encoder_config.number_of_streams = 1;
3959 video_encoder_config.video_stream_factory =
3960 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3961 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003962 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003963 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3964
3965 // Detector should be updated with fps limit from codec config.
3966 EXPECT_EQ(
3967 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3968 kFramerate);
3969
3970 // Trigger overuse, max framerate should be reduced.
3971 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3972 stats.input_frame_rate = kFramerate;
3973 stats_proxy_->SetMockStats(stats);
3974 video_stream_encoder_->TriggerCpuOveruse();
3975 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3976 int adapted_framerate =
3977 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3978 EXPECT_LT(adapted_framerate, kFramerate);
3979
3980 // Trigger underuse, max framerate should go back to codec configured fps.
3981 // Set extra low fps, to make sure it's actually reset, not just incremented.
3982 stats = stats_proxy_->GetStats();
3983 stats.input_frame_rate = adapted_framerate / 2;
3984 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003985 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003986 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3987 EXPECT_EQ(
3988 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3989 kFramerate);
3990
3991 video_stream_encoder_->Stop();
3992}
3993
3994TEST_F(VideoStreamEncoderTest,
3995 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3996 const int kFrameWidth = 1280;
3997 const int kFrameHeight = 720;
3998 const int kLowFramerate = 15;
3999 const int kHighFramerate = 25;
4000
Henrik Boström381d1092020-05-12 18:49:07 +02004001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004002 DataRate::BitsPerSec(kTargetBitrateBps),
4003 DataRate::BitsPerSec(kTargetBitrateBps),
4004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004005 test::FrameForwarder source;
4006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004007 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004008
4009 // Trigger initial configuration.
4010 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004011 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01004012 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4013 video_encoder_config.number_of_streams = 1;
4014 video_encoder_config.video_stream_factory =
4015 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
4016 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4017 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004018 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004019 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4020
4021 EXPECT_EQ(
4022 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4023 kLowFramerate);
4024
4025 // Trigger overuse, max framerate should be reduced.
4026 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4027 stats.input_frame_rate = kLowFramerate;
4028 stats_proxy_->SetMockStats(stats);
4029 video_stream_encoder_->TriggerCpuOveruse();
4030 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4031 int adapted_framerate =
4032 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4033 EXPECT_LT(adapted_framerate, kLowFramerate);
4034
4035 // Reconfigure the encoder with a new (higher max framerate), max fps should
4036 // still respect the adaptation.
4037 video_encoder_config.video_stream_factory =
4038 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
4039 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4040 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004041 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004042 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4043
4044 EXPECT_EQ(
4045 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4046 adapted_framerate);
4047
4048 // Trigger underuse, max framerate should go back to codec configured fps.
4049 stats = stats_proxy_->GetStats();
4050 stats.input_frame_rate = adapted_framerate;
4051 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004052 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004053 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4054 EXPECT_EQ(
4055 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4056 kHighFramerate);
4057
4058 video_stream_encoder_->Stop();
4059}
4060
mflodmancc3d4422017-08-03 08:27:51 -07004061TEST_F(VideoStreamEncoderTest,
4062 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07004063 const int kFrameWidth = 1280;
4064 const int kFrameHeight = 720;
4065 const int kFramerate = 24;
4066
Henrik Boström381d1092020-05-12 18:49:07 +02004067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004068 DataRate::BitsPerSec(kTargetBitrateBps),
4069 DataRate::BitsPerSec(kTargetBitrateBps),
4070 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07004071 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004072 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004073 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07004074
4075 // Trigger initial configuration.
4076 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004077 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07004078 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4079 video_encoder_config.number_of_streams = 1;
4080 video_encoder_config.video_stream_factory =
4081 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
4082 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004083 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004084 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004085 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004086
Niels Möller7dc26b72017-12-06 10:27:48 +01004087 EXPECT_EQ(
4088 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4089 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004090
4091 // Trigger overuse, max framerate should be reduced.
4092 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4093 stats.input_frame_rate = kFramerate;
4094 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004095 video_stream_encoder_->TriggerCpuOveruse();
4096 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004097 int adapted_framerate =
4098 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004099 EXPECT_LT(adapted_framerate, kFramerate);
4100
4101 // Change degradation preference to not enable framerate scaling. Target
4102 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004103 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004104 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004105 EXPECT_EQ(
4106 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4107 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004108
mflodmancc3d4422017-08-03 08:27:51 -07004109 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004110}
4111
mflodmancc3d4422017-08-03 08:27:51 -07004112TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004113 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004115 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4116 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4117 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004118 const int kWidth = 640;
4119 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004120
asaperssonfab67072017-04-04 05:51:49 -07004121 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004122
4123 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004124 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004125
4126 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004127 EXPECT_TRUE_WAIT(
4128 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004129
sprangc5d62e22017-04-02 23:53:04 -07004130 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004131
asaperssonfab67072017-04-04 05:51:49 -07004132 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004133 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004134 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004135
4136 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004137 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004138
Henrik Boström2671dac2020-05-19 16:29:09 +02004139 EXPECT_TRUE_WAIT(
4140 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004141
mflodmancc3d4422017-08-03 08:27:51 -07004142 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004143}
4144
mflodmancc3d4422017-08-03 08:27:51 -07004145TEST_F(VideoStreamEncoderTest,
4146 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004147 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004148 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004149 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4150 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4151 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004152 const int kWidth = 640;
4153 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004154
4155 // We expect the n initial frames to get dropped.
4156 int i;
4157 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004158 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004159 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004160 }
4161 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004162 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004163 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004164
4165 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004166 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004167
mflodmancc3d4422017-08-03 08:27:51 -07004168 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004169}
4170
mflodmancc3d4422017-08-03 08:27:51 -07004171TEST_F(VideoStreamEncoderTest,
4172 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004173 const int kWidth = 640;
4174 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004176 DataRate::BitsPerSec(kLowTargetBitrateBps),
4177 DataRate::BitsPerSec(kLowTargetBitrateBps),
4178 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004179
4180 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004181 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004182 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004183
asaperssonfab67072017-04-04 05:51:49 -07004184 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004185 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004186 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004187
mflodmancc3d4422017-08-03 08:27:51 -07004188 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004189}
4190
mflodmancc3d4422017-08-03 08:27:51 -07004191TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004192 const int kWidth = 640;
4193 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004194 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004195
4196 VideoEncoderConfig video_encoder_config;
4197 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4198 // Make format different, to force recreation of encoder.
4199 video_encoder_config.video_format.parameters["foo"] = "foo";
4200 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004201 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004203 DataRate::BitsPerSec(kLowTargetBitrateBps),
4204 DataRate::BitsPerSec(kLowTargetBitrateBps),
4205 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004206
kthelgasonb83797b2017-02-14 11:57:25 -08004207 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004208 video_stream_encoder_->SetSource(&video_source_,
4209 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004210
asaperssonfab67072017-04-04 05:51:49 -07004211 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004212 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004213 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004214
mflodmancc3d4422017-08-03 08:27:51 -07004215 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004216 fake_encoder_.SetQualityScaling(true);
4217}
4218
Åsa Persson139f4dc2019-08-02 09:29:58 +02004219TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4220 webrtc::test::ScopedFieldTrials field_trials(
4221 "WebRTC-Video-QualityScalerSettings/"
4222 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4223 // Reset encoder for field trials to take effect.
4224 ConfigureEncoder(video_encoder_config_.Copy());
4225 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4226 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4227 const int kWidth = 640;
4228 const int kHeight = 360;
4229
Henrik Boström381d1092020-05-12 18:49:07 +02004230 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004231 DataRate::BitsPerSec(kTargetBitrateBps),
4232 DataRate::BitsPerSec(kTargetBitrateBps),
4233 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004234 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4235 // Frame should not be dropped.
4236 WaitForEncodedFrame(1);
4237
Henrik Boström381d1092020-05-12 18:49:07 +02004238 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004239 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4240 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4241 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004242 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4243 // Frame should not be dropped.
4244 WaitForEncodedFrame(2);
4245
Henrik Boström381d1092020-05-12 18:49:07 +02004246 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004247 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4248 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4249 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004250 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4251 // Expect to drop this frame, the wait should time out.
4252 ExpectDroppedFrame();
4253
4254 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004255 EXPECT_TRUE_WAIT(
4256 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004257 video_stream_encoder_->Stop();
4258}
4259
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004260TEST_F(VideoStreamEncoderTest,
4261 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
4262 webrtc::test::ScopedFieldTrials field_trials(
4263 "WebRTC-Video-QualityScalerSettings/"
4264 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4265 fake_encoder_.SetQualityScaling(false);
4266 ConfigureEncoder(video_encoder_config_.Copy());
4267 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4268 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4269 const int kWidth = 640;
4270 const int kHeight = 360;
4271
4272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4273 DataRate::BitsPerSec(kTargetBitrateBps),
4274 DataRate::BitsPerSec(kTargetBitrateBps),
4275 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4276 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4277 // Frame should not be dropped.
4278 WaitForEncodedFrame(1);
4279
4280 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4281 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4282 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4283 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
4284 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4285 // Frame should not be dropped.
4286 WaitForEncodedFrame(2);
4287
4288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4289 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4290 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4291 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
4292 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4293 // Not dropped since quality scaling is disabled.
4294 WaitForEncodedFrame(3);
4295
4296 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02004297 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004298 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
4299
4300 video_stream_encoder_->Stop();
4301}
4302
Åsa Perssone644a032019-11-08 15:56:00 +01004303TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4304 webrtc::test::ScopedFieldTrials field_trials(
4305 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4306
4307 // Reset encoder for field trials to take effect.
4308 VideoEncoderConfig config = video_encoder_config_.Copy();
4309 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004310 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004311 ConfigureEncoder(std::move(config));
4312 fake_encoder_.SetQp(kQpLow);
4313
4314 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004315 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01004316 source.set_adaptation_enabled(true);
4317 video_stream_encoder_->SetSource(&source,
4318 DegradationPreference::MAINTAIN_FRAMERATE);
4319
4320 // Start at low bitrate.
4321 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004322 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4323 DataRate::BitsPerSec(kLowBitrateBps),
4324 DataRate::BitsPerSec(kLowBitrateBps),
4325 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004326
4327 // Expect first frame to be dropped and resolution to be limited.
4328 const int kWidth = 1280;
4329 const int kHeight = 720;
4330 const int64_t kFrameIntervalMs = 100;
4331 int64_t timestamp_ms = kFrameIntervalMs;
4332 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4333 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004334 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4335 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004336
4337 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4339 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004340
4341 // Insert frames and advance |min_duration_ms|.
4342 for (size_t i = 1; i <= 10; i++) {
4343 timestamp_ms += kFrameIntervalMs;
4344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4345 WaitForEncodedFrame(timestamp_ms);
4346 }
4347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4348 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4349
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004350 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004351
4352 // Insert frame should trigger high BW and release quality limitation.
4353 timestamp_ms += kFrameIntervalMs;
4354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4355 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004356 // The ramp-up code involves the adaptation queue, give it time to execute.
4357 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02004358 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004359 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004360
4361 // Frame should not be adapted.
4362 timestamp_ms += kFrameIntervalMs;
4363 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4364 WaitForEncodedFrame(kWidth, kHeight);
4365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4366
4367 video_stream_encoder_->Stop();
4368}
4369
mflodmancc3d4422017-08-03 08:27:51 -07004370TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004371 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004372 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004373 source.set_adaptation_enabled(true);
4374 video_stream_encoder_->SetSource(&source,
4375 DegradationPreference::MAINTAIN_FRAMERATE);
4376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4377 DataRate::BitsPerSec(kTargetBitrateBps),
4378 DataRate::BitsPerSec(kTargetBitrateBps),
4379 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4380 fake_encoder_.SetQp(kQpHigh + 1);
4381 const int kWidth = 1280;
4382 const int kHeight = 720;
4383 const int64_t kFrameIntervalMs = 100;
4384 int64_t timestamp_ms = kFrameIntervalMs;
4385 for (size_t i = 1; i <= 100; i++) {
4386 timestamp_ms += kFrameIntervalMs;
4387 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4388 WaitForEncodedFrame(timestamp_ms);
4389 }
4390 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
4391 // for the first time.
4392 // TODO(eshr): We should avoid these waits by using threads with simulated
4393 // time.
4394 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
4395 2000 * 2.5 * 2);
4396 timestamp_ms += kFrameIntervalMs;
4397 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4398 WaitForEncodedFrame(timestamp_ms);
4399 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4400 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
4401 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4402
4403 // Disable Quality scaling by turning off scaler on the encoder and
4404 // reconfiguring.
4405 fake_encoder_.SetQualityScaling(false);
4406 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
4407 kMaxPayloadLength);
4408 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004409 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02004410 // Since we turned off the quality scaler, the adaptations made by it are
4411 // removed.
4412 EXPECT_THAT(source.sink_wants(), ResolutionMax());
4413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4414
4415 video_stream_encoder_->Stop();
4416}
4417
4418TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004419 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4420 const int kTooSmallWidth = 10;
4421 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004423 DataRate::BitsPerSec(kTargetBitrateBps),
4424 DataRate::BitsPerSec(kTargetBitrateBps),
4425 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004426
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004427 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004428 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004429 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004430 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004431 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004432 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4433
4434 // Trigger adapt down, too small frame, expect no change.
4435 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004436 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004437 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004438 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004439 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4440 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4441
mflodmancc3d4422017-08-03 08:27:51 -07004442 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004443}
4444
mflodmancc3d4422017-08-03 08:27:51 -07004445TEST_F(VideoStreamEncoderTest,
4446 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004447 const int kTooSmallWidth = 10;
4448 const int kTooSmallHeight = 10;
4449 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004450 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004451 DataRate::BitsPerSec(kTargetBitrateBps),
4452 DataRate::BitsPerSec(kTargetBitrateBps),
4453 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004454
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004455 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004456 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004457 video_stream_encoder_->SetSource(&source,
4458 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004459 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004460 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4461 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4462
4463 // Trigger adapt down, expect limited framerate.
4464 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004465 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004466 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004467 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004468 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4470 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4471
4472 // Trigger adapt down, too small frame, expect no change.
4473 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004474 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004475 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004476 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4479 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4480
mflodmancc3d4422017-08-03 08:27:51 -07004481 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004482}
4483
mflodmancc3d4422017-08-03 08:27:51 -07004484TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004485 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004487 DataRate::BitsPerSec(kTargetBitrateBps),
4488 DataRate::BitsPerSec(kTargetBitrateBps),
4489 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004490 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004491 const int kFrameWidth = 1280;
4492 const int kFrameHeight = 720;
4493 video_source_.IncomingCapturedFrame(
4494 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004495 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004496 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004497}
4498
sprangb1ca0732017-02-01 08:38:12 -08004499// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004500TEST_F(VideoStreamEncoderTest,
4501 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004503 DataRate::BitsPerSec(kTargetBitrateBps),
4504 DataRate::BitsPerSec(kTargetBitrateBps),
4505 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004506
4507 const int kFrameWidth = 1280;
4508 const int kFrameHeight = 720;
4509 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004510 // requested by
4511 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004512 video_source_.set_adaptation_enabled(true);
4513
4514 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004515 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004516 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004517
4518 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004519 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004520 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004521 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004522 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004523
asaperssonfab67072017-04-04 05:51:49 -07004524 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004525 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004526 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004527 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004528 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004529
mflodmancc3d4422017-08-03 08:27:51 -07004530 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004531}
sprangfe627f32017-03-29 08:24:59 -07004532
mflodmancc3d4422017-08-03 08:27:51 -07004533TEST_F(VideoStreamEncoderTest,
4534 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004535 const int kFrameWidth = 1280;
4536 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004537
Henrik Boström381d1092020-05-12 18:49:07 +02004538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004539 DataRate::BitsPerSec(kTargetBitrateBps),
4540 DataRate::BitsPerSec(kTargetBitrateBps),
4541 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004542 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004543 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004544 video_source_.set_adaptation_enabled(true);
4545
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004546 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07004547
4548 video_source_.IncomingCapturedFrame(
4549 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004550 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004551
4552 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004553 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004554
4555 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004556 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004557 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004558 video_source_.IncomingCapturedFrame(
4559 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004560 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004561 }
4562
4563 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004564 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004565 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004566 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004567 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004568 video_source_.IncomingCapturedFrame(
4569 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004570 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004571 ++num_frames_dropped;
4572 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004573 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004574 }
4575 }
4576
sprang4847ae62017-06-27 07:06:52 -07004577 // Add some slack to account for frames dropped by the frame dropper.
4578 const int kErrorMargin = 1;
4579 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004580 kErrorMargin);
4581
4582 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004583 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004584 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004585 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004586 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004587 video_source_.IncomingCapturedFrame(
4588 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004589 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004590 ++num_frames_dropped;
4591 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004592 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004593 }
4594 }
sprang4847ae62017-06-27 07:06:52 -07004595 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004596 kErrorMargin);
4597
4598 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004599 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004600 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004601 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004602 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004603 video_source_.IncomingCapturedFrame(
4604 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004605 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004606 ++num_frames_dropped;
4607 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004608 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004609 }
4610 }
sprang4847ae62017-06-27 07:06:52 -07004611 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004612 kErrorMargin);
4613
4614 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004615 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004616 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004617 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004618 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004619 video_source_.IncomingCapturedFrame(
4620 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004621 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004622 ++num_frames_dropped;
4623 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004624 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004625 }
4626 }
4627 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4628
mflodmancc3d4422017-08-03 08:27:51 -07004629 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004630}
4631
mflodmancc3d4422017-08-03 08:27:51 -07004632TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004633 const int kFramerateFps = 5;
4634 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004635 const int kFrameWidth = 1280;
4636 const int kFrameHeight = 720;
4637
sprang4847ae62017-06-27 07:06:52 -07004638 // Reconfigure encoder with two temporal layers and screensharing, which will
4639 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004640 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004641
Henrik Boström381d1092020-05-12 18:49:07 +02004642 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004643 DataRate::BitsPerSec(kTargetBitrateBps),
4644 DataRate::BitsPerSec(kTargetBitrateBps),
4645 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004647 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004648 video_source_.set_adaptation_enabled(true);
4649
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004650 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07004651
4652 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004653 rtc::VideoSinkWants last_wants;
4654 do {
4655 last_wants = video_source_.sink_wants();
4656
sprangc5d62e22017-04-02 23:53:04 -07004657 // Insert frames to get a new fps estimate...
4658 for (int j = 0; j < kFramerateFps; ++j) {
4659 video_source_.IncomingCapturedFrame(
4660 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004661 if (video_source_.last_sent_width()) {
4662 sink_.WaitForEncodedFrame(timestamp_ms);
4663 }
sprangc5d62e22017-04-02 23:53:04 -07004664 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004665 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004666 }
4667 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004668 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004669 } while (video_source_.sink_wants().max_framerate_fps <
4670 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004671
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004672 EXPECT_THAT(video_source_.sink_wants(),
4673 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004674
mflodmancc3d4422017-08-03 08:27:51 -07004675 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004676}
asaperssonf7e294d2017-06-13 23:25:22 -07004677
mflodmancc3d4422017-08-03 08:27:51 -07004678TEST_F(VideoStreamEncoderTest,
4679 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004680 const int kWidth = 1280;
4681 const int kHeight = 720;
4682 const int64_t kFrameIntervalMs = 150;
4683 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004685 DataRate::BitsPerSec(kTargetBitrateBps),
4686 DataRate::BitsPerSec(kTargetBitrateBps),
4687 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004688
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004689 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004690 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07004691 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004692 video_stream_encoder_->SetSource(&source,
4693 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004694 timestamp_ms += kFrameIntervalMs;
4695 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004696 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004697 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004698 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4699 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4700 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4701
4702 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004703 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004704 timestamp_ms += kFrameIntervalMs;
4705 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004706 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004707 EXPECT_THAT(source.sink_wants(),
4708 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004709 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4711 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4712
4713 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004714 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004715 timestamp_ms += kFrameIntervalMs;
4716 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004717 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004719 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4720 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4721 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4722
4723 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004724 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004725 timestamp_ms += kFrameIntervalMs;
4726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004727 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004728 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004729 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4730 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4731 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4732
4733 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004734 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004735 timestamp_ms += kFrameIntervalMs;
4736 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004737 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004738 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004739 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4740 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4741 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4742
4743 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004744 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004745 timestamp_ms += kFrameIntervalMs;
4746 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004747 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004748 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004749 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4750 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4751 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4752
4753 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004754 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004755 timestamp_ms += kFrameIntervalMs;
4756 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004757 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004758 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004759 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4760 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4761 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4762
4763 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004764 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004765 timestamp_ms += kFrameIntervalMs;
4766 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004767 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004768 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004769 rtc::VideoSinkWants last_wants = source.sink_wants();
4770 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4771 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4772 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4773
4774 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004775 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004776 timestamp_ms += kFrameIntervalMs;
4777 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004778 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004779 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004780 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4781 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4782 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4783
Evan Shrubsole64469032020-06-11 10:45:29 +02004784 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004785 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004786 timestamp_ms += kFrameIntervalMs;
4787 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004788 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004789 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004790 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4791 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4792 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4793
4794 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004795 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004796 timestamp_ms += kFrameIntervalMs;
4797 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004798 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004799 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004800 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4801 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4802 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4803
4804 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004805 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004806 timestamp_ms += kFrameIntervalMs;
4807 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004808 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004809 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004810 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4811 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4812 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4813
4814 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004815 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004816 timestamp_ms += kFrameIntervalMs;
4817 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004818 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004819 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004820 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4821 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4822 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4823
4824 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004825 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004826 timestamp_ms += kFrameIntervalMs;
4827 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004828 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004829 EXPECT_THAT(source.sink_wants(), FpsMax());
4830 EXPECT_EQ(source.sink_wants().max_pixel_count,
4831 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004832 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4833 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4834 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4835
4836 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004837 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004838 timestamp_ms += kFrameIntervalMs;
4839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004840 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004841 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004842 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4844 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4845
Åsa Persson30ab0152019-08-27 12:22:33 +02004846 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004847 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004848 timestamp_ms += kFrameIntervalMs;
4849 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004850 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004851 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004852 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4854 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4855 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4856
4857 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004858 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004859 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004860 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4861
mflodmancc3d4422017-08-03 08:27:51 -07004862 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004863}
4864
mflodmancc3d4422017-08-03 08:27:51 -07004865TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004866 const int kWidth = 1280;
4867 const int kHeight = 720;
4868 const int64_t kFrameIntervalMs = 150;
4869 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004871 DataRate::BitsPerSec(kTargetBitrateBps),
4872 DataRate::BitsPerSec(kTargetBitrateBps),
4873 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004874
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004875 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004876 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07004877 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004878 video_stream_encoder_->SetSource(&source,
4879 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004880 timestamp_ms += kFrameIntervalMs;
4881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004882 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004883 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004884 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4885 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4886 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4887 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4888 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4889 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4890
4891 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004892 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004893 timestamp_ms += kFrameIntervalMs;
4894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004895 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004896 EXPECT_THAT(source.sink_wants(),
4897 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4899 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4900 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4901 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4902 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4903 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4904
4905 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004906 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004907 timestamp_ms += kFrameIntervalMs;
4908 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004909 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004910 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004911 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4912 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4913 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4914 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4915 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4916 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4917
4918 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004919 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004920 timestamp_ms += kFrameIntervalMs;
4921 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004922 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004923 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02004924 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07004925 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4926 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4927 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4928 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4929 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4930
Evan Shrubsole64469032020-06-11 10:45:29 +02004931 // Trigger cpu adapt up, expect no change since QP is most limited.
4932 {
4933 // Store current sink wants since we expect no change and if there is no
4934 // change then last_wants() is not updated.
4935 auto previous_sink_wants = source.sink_wants();
4936 video_stream_encoder_->TriggerCpuUnderuse();
4937 timestamp_ms += kFrameIntervalMs;
4938 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4939 WaitForEncodedFrame(timestamp_ms);
4940 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4941 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4942 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4943 }
4944
4945 // Trigger quality adapt up, expect increased fps (640x360@30fps).
4946 video_stream_encoder_->TriggerQualityHigh();
4947 timestamp_ms += kFrameIntervalMs;
4948 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4949 WaitForEncodedFrame(timestamp_ms);
4950 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
4951 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4952 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4953 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4954 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4955 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4956 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4957
4958 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4959 // expect increased resolution (960x540@30fps).
4960 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004961 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004962 timestamp_ms += kFrameIntervalMs;
4963 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004964 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02004965 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004966 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4967 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4968 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4969 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4970 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004971 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004972
Evan Shrubsole64469032020-06-11 10:45:29 +02004973 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4974 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004975 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004976 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004977 timestamp_ms += kFrameIntervalMs;
4978 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004979 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004980 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004981 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004982 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4983 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4984 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4985 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4986 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004987 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004988
4989 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004990 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004991 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004992 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004993 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004994
mflodmancc3d4422017-08-03 08:27:51 -07004995 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004996}
4997
mflodmancc3d4422017-08-03 08:27:51 -07004998TEST_F(VideoStreamEncoderTest,
4999 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07005000 const int kWidth = 640;
5001 const int kHeight = 360;
5002 const int kFpsLimit = 15;
5003 const int64_t kFrameIntervalMs = 150;
5004 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005005 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005006 DataRate::BitsPerSec(kTargetBitrateBps),
5007 DataRate::BitsPerSec(kTargetBitrateBps),
5008 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005009
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005010 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005011 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005012 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005013 video_stream_encoder_->SetSource(&source,
5014 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005015 timestamp_ms += kFrameIntervalMs;
5016 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005017 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005018 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005019 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5020 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5021 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5022 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5023 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5024 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5025
5026 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005027 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07005028 timestamp_ms += kFrameIntervalMs;
5029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005030 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005031 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5033 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5034 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5035 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5036 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5037 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5038
5039 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005040 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005041 timestamp_ms += kFrameIntervalMs;
5042 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005043 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005044 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005045 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02005046 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07005047 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5048 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5049 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5050 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5051
Evan Shrubsole64469032020-06-11 10:45:29 +02005052 // Trigger cpu adapt up, expect no change because quality is most limited.
5053 {
5054 auto previous_sink_wants = source.sink_wants();
5055 // Store current sink wants since we expect no change ind if there is no
5056 // change then last__wants() is not updated.
5057 video_stream_encoder_->TriggerCpuUnderuse();
5058 timestamp_ms += kFrameIntervalMs;
5059 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5060 WaitForEncodedFrame(timestamp_ms);
5061 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
5062 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5063 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5064 }
5065
5066 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
5067 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005068 timestamp_ms += kFrameIntervalMs;
5069 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005070 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005071 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005072 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5073 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5074 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02005075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
5076 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5077 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005078
Evan Shrubsole64469032020-06-11 10:45:29 +02005079 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005080 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02005081 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07005082 timestamp_ms += kFrameIntervalMs;
5083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005084 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005085 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005086 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5087 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5088 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5089 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
5090 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005091 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005092
5093 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005094 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005095 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005096 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02005097 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07005098
mflodmancc3d4422017-08-03 08:27:51 -07005099 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005100}
5101
mflodmancc3d4422017-08-03 08:27:51 -07005102TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07005103 const int kFrameWidth = 1920;
5104 const int kFrameHeight = 1080;
5105 // 3/4 of 1920.
5106 const int kAdaptedFrameWidth = 1440;
5107 // 3/4 of 1080 rounded down to multiple of 4.
5108 const int kAdaptedFrameHeight = 808;
5109 const int kFramerate = 24;
5110
Henrik Boström381d1092020-05-12 18:49:07 +02005111 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005112 DataRate::BitsPerSec(kTargetBitrateBps),
5113 DataRate::BitsPerSec(kTargetBitrateBps),
5114 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07005115 // Trigger reconfigure encoder (without resetting the entire instance).
5116 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02005117 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07005118 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
5119 video_encoder_config.number_of_streams = 1;
5120 video_encoder_config.video_stream_factory =
5121 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07005122 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005123 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005124 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07005125
5126 video_source_.set_adaptation_enabled(true);
5127
5128 video_source_.IncomingCapturedFrame(
5129 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005130 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005131
5132 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005133 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07005134 video_source_.IncomingCapturedFrame(
5135 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005136 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005137
mflodmancc3d4422017-08-03 08:27:51 -07005138 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07005139}
5140
mflodmancc3d4422017-08-03 08:27:51 -07005141TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07005142 const int kFrameWidth = 1280;
5143 const int kFrameHeight = 720;
5144 const int kLowFps = 2;
5145 const int kHighFps = 30;
5146
Henrik Boström381d1092020-05-12 18:49:07 +02005147 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005148 DataRate::BitsPerSec(kTargetBitrateBps),
5149 DataRate::BitsPerSec(kTargetBitrateBps),
5150 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005151
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005152 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07005153 max_framerate_ = kLowFps;
5154
5155 // Insert 2 seconds of 2fps video.
5156 for (int i = 0; i < kLowFps * 2; ++i) {
5157 video_source_.IncomingCapturedFrame(
5158 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5159 WaitForEncodedFrame(timestamp_ms);
5160 timestamp_ms += 1000 / kLowFps;
5161 }
5162
5163 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02005164 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005165 DataRate::BitsPerSec(kTargetBitrateBps),
5166 DataRate::BitsPerSec(kTargetBitrateBps),
5167 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005168 video_source_.IncomingCapturedFrame(
5169 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5170 WaitForEncodedFrame(timestamp_ms);
5171 timestamp_ms += 1000 / kLowFps;
5172
5173 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
5174
5175 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02005176 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07005177 const int kFrameIntervalMs = 1000 / kHighFps;
5178 max_framerate_ = kHighFps;
5179 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
5180 video_source_.IncomingCapturedFrame(
5181 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5182 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
5183 // be dropped if the encoder hans't been updated with the new higher target
5184 // framerate yet, causing it to overshoot the target bitrate and then
5185 // suffering the wrath of the media optimizer.
5186 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
5187 timestamp_ms += kFrameIntervalMs;
5188 }
5189
5190 // Don expect correct measurement just yet, but it should be higher than
5191 // before.
5192 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
5193
mflodmancc3d4422017-08-03 08:27:51 -07005194 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005195}
5196
mflodmancc3d4422017-08-03 08:27:51 -07005197TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07005198 const int kFrameWidth = 1280;
5199 const int kFrameHeight = 720;
5200 const int kTargetBitrateBps = 1000000;
5201
5202 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02005203 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02005204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005205 DataRate::BitsPerSec(kTargetBitrateBps),
5206 DataRate::BitsPerSec(kTargetBitrateBps),
5207 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005208 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005209
5210 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005211 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07005212 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5213 video_source_.IncomingCapturedFrame(
5214 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5215 WaitForEncodedFrame(timestamp_ms);
5216
5217 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5219 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5220 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005221
5222 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005223 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005224 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005225
5226 // Bitrate observer should not be called.
5227 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5228 video_source_.IncomingCapturedFrame(
5229 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5230 ExpectDroppedFrame();
5231
mflodmancc3d4422017-08-03 08:27:51 -07005232 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005233}
ilnik6b826ef2017-06-16 06:53:48 -07005234
Niels Möller4db138e2018-04-19 09:04:13 +02005235TEST_F(VideoStreamEncoderTest,
5236 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5237 const int kFrameWidth = 1280;
5238 const int kFrameHeight = 720;
5239 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005240 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005241 DataRate::BitsPerSec(kTargetBitrateBps),
5242 DataRate::BitsPerSec(kTargetBitrateBps),
5243 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005244 video_source_.IncomingCapturedFrame(
5245 CreateFrame(1, kFrameWidth, kFrameHeight));
5246 WaitForEncodedFrame(1);
5247 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5248 .low_encode_usage_threshold_percent,
5249 default_options.low_encode_usage_threshold_percent);
5250 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5251 .high_encode_usage_threshold_percent,
5252 default_options.high_encode_usage_threshold_percent);
5253 video_stream_encoder_->Stop();
5254}
5255
5256TEST_F(VideoStreamEncoderTest,
5257 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5258 const int kFrameWidth = 1280;
5259 const int kFrameHeight = 720;
5260 CpuOveruseOptions hardware_options;
5261 hardware_options.low_encode_usage_threshold_percent = 150;
5262 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005263 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005264
Henrik Boström381d1092020-05-12 18:49:07 +02005265 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005266 DataRate::BitsPerSec(kTargetBitrateBps),
5267 DataRate::BitsPerSec(kTargetBitrateBps),
5268 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005269 video_source_.IncomingCapturedFrame(
5270 CreateFrame(1, kFrameWidth, kFrameHeight));
5271 WaitForEncodedFrame(1);
5272 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5273 .low_encode_usage_threshold_percent,
5274 hardware_options.low_encode_usage_threshold_percent);
5275 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5276 .high_encode_usage_threshold_percent,
5277 hardware_options.high_encode_usage_threshold_percent);
5278 video_stream_encoder_->Stop();
5279}
5280
Niels Möller6bb5ab92019-01-11 11:11:10 +01005281TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5282 const int kFrameWidth = 320;
5283 const int kFrameHeight = 240;
5284 const int kFps = 30;
5285 const int kTargetBitrateBps = 120000;
5286 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5287
Henrik Boström381d1092020-05-12 18:49:07 +02005288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005289 DataRate::BitsPerSec(kTargetBitrateBps),
5290 DataRate::BitsPerSec(kTargetBitrateBps),
5291 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005292
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005293 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01005294 max_framerate_ = kFps;
5295
5296 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5297 fake_encoder_.SimulateOvershoot(1.0);
5298 int num_dropped = 0;
5299 for (int i = 0; i < kNumFramesInRun; ++i) {
5300 video_source_.IncomingCapturedFrame(
5301 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5302 // Wait up to two frame durations for a frame to arrive.
5303 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5304 ++num_dropped;
5305 }
5306 timestamp_ms += 1000 / kFps;
5307 }
5308
Erik Språnga8d48ab2019-02-08 14:17:40 +01005309 // Framerate should be measured to be near the expected target rate.
5310 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5311
5312 // Frame drops should be within 5% of expected 0%.
5313 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005314
5315 // Make encoder produce frames at double the expected bitrate during 3 seconds
5316 // of video, verify number of drops. Rate needs to be slightly changed in
5317 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005318 double overshoot_factor = 2.0;
5319 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5320 // With bitrate adjuster, when need to overshoot even more to trigger
5321 // frame dropping.
5322 overshoot_factor *= 2;
5323 }
5324 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005326 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5327 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5328 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005329 num_dropped = 0;
5330 for (int i = 0; i < kNumFramesInRun; ++i) {
5331 video_source_.IncomingCapturedFrame(
5332 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5333 // Wait up to two frame durations for a frame to arrive.
5334 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5335 ++num_dropped;
5336 }
5337 timestamp_ms += 1000 / kFps;
5338 }
5339
Henrik Boström381d1092020-05-12 18:49:07 +02005340 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005341 DataRate::BitsPerSec(kTargetBitrateBps),
5342 DataRate::BitsPerSec(kTargetBitrateBps),
5343 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005344
5345 // Target framerate should be still be near the expected target, despite
5346 // the frame drops.
5347 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5348
5349 // Frame drops should be within 5% of expected 50%.
5350 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005351
5352 video_stream_encoder_->Stop();
5353}
5354
5355TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5356 const int kFrameWidth = 320;
5357 const int kFrameHeight = 240;
5358 const int kActualInputFps = 24;
5359 const int kTargetBitrateBps = 120000;
5360
5361 ASSERT_GT(max_framerate_, kActualInputFps);
5362
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005363 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01005364 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005365 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005366 DataRate::BitsPerSec(kTargetBitrateBps),
5367 DataRate::BitsPerSec(kTargetBitrateBps),
5368 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005369
5370 // Insert 3 seconds of video, with an input fps lower than configured max.
5371 for (int i = 0; i < kActualInputFps * 3; ++i) {
5372 video_source_.IncomingCapturedFrame(
5373 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5374 // Wait up to two frame durations for a frame to arrive.
5375 WaitForEncodedFrame(timestamp_ms);
5376 timestamp_ms += 1000 / kActualInputFps;
5377 }
5378
5379 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5380
5381 video_stream_encoder_->Stop();
5382}
5383
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005384TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005385 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005387 DataRate::BitsPerSec(kTargetBitrateBps),
5388 DataRate::BitsPerSec(kTargetBitrateBps),
5389 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005390
5391 fake_encoder_.BlockNextEncode();
5392 video_source_.IncomingCapturedFrame(
5393 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5394 WaitForEncodedFrame(1);
5395 // On the very first frame full update should be forced.
5396 rect = fake_encoder_.GetLastUpdateRect();
5397 EXPECT_EQ(rect.offset_x, 0);
5398 EXPECT_EQ(rect.offset_y, 0);
5399 EXPECT_EQ(rect.height, codec_height_);
5400 EXPECT_EQ(rect.width, codec_width_);
5401 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5402 // call to ContinueEncode.
5403 video_source_.IncomingCapturedFrame(
5404 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5405 ExpectDroppedFrame();
5406 video_source_.IncomingCapturedFrame(
5407 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5408 ExpectDroppedFrame();
5409 fake_encoder_.ContinueEncode();
5410 WaitForEncodedFrame(3);
5411 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5412 rect = fake_encoder_.GetLastUpdateRect();
5413 EXPECT_EQ(rect.offset_x, 1);
5414 EXPECT_EQ(rect.offset_y, 0);
5415 EXPECT_EQ(rect.width, 10);
5416 EXPECT_EQ(rect.height, 1);
5417
5418 video_source_.IncomingCapturedFrame(
5419 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5420 WaitForEncodedFrame(4);
5421 // Previous frame was encoded, so no accumulation should happen.
5422 rect = fake_encoder_.GetLastUpdateRect();
5423 EXPECT_EQ(rect.offset_x, 0);
5424 EXPECT_EQ(rect.offset_y, 0);
5425 EXPECT_EQ(rect.width, 1);
5426 EXPECT_EQ(rect.height, 1);
5427
5428 video_stream_encoder_->Stop();
5429}
5430
Erik Språngd7329ca2019-02-21 21:19:53 +01005431TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005432 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005433 DataRate::BitsPerSec(kTargetBitrateBps),
5434 DataRate::BitsPerSec(kTargetBitrateBps),
5435 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005436
5437 // First frame is always keyframe.
5438 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5439 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005440 EXPECT_THAT(
5441 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005442 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005443
5444 // Insert delta frame.
5445 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5446 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005447 EXPECT_THAT(
5448 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005449 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005450
5451 // Request next frame be a key-frame.
5452 video_stream_encoder_->SendKeyFrame();
5453 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5454 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005455 EXPECT_THAT(
5456 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005457 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005458
5459 video_stream_encoder_->Stop();
5460}
5461
5462TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5463 // Setup simulcast with three streams.
5464 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005465 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005466 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5467 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5468 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005469 // Wait for all three layers before triggering event.
5470 sink_.SetNumExpectedLayers(3);
5471
5472 // First frame is always keyframe.
5473 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5474 WaitForEncodedFrame(1);
5475 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005476 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5477 VideoFrameType::kVideoFrameKey,
5478 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005479
5480 // Insert delta frame.
5481 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5482 WaitForEncodedFrame(2);
5483 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005484 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5485 VideoFrameType::kVideoFrameDelta,
5486 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005487
5488 // Request next frame be a key-frame.
5489 // Only first stream is configured to produce key-frame.
5490 video_stream_encoder_->SendKeyFrame();
5491 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5492 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005493
5494 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5495 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005496 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005497 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005498 VideoFrameType::kVideoFrameKey,
5499 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005500
5501 video_stream_encoder_->Stop();
5502}
5503
5504TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5505 // Configure internal source factory and setup test again.
5506 encoder_factory_.SetHasInternalSource(true);
5507 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005509 DataRate::BitsPerSec(kTargetBitrateBps),
5510 DataRate::BitsPerSec(kTargetBitrateBps),
5511 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005512
5513 // Call encoder directly, simulating internal source where encoded frame
5514 // callback in VideoStreamEncoder is called despite no OnFrame().
5515 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5516 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005517 EXPECT_THAT(
5518 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005519 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005520
Niels Möller8f7ce222019-03-21 15:43:58 +01005521 const std::vector<VideoFrameType> kDeltaFrame = {
5522 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005523 // Need to set timestamp manually since manually for injected frame.
5524 VideoFrame frame = CreateFrame(101, nullptr);
5525 frame.set_timestamp(101);
5526 fake_encoder_.InjectFrame(frame, false);
5527 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005528 EXPECT_THAT(
5529 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005530 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005531
5532 // Request key-frame. The forces a dummy frame down into the encoder.
5533 fake_encoder_.ExpectNullFrame();
5534 video_stream_encoder_->SendKeyFrame();
5535 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005536 EXPECT_THAT(
5537 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005538 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005539
5540 video_stream_encoder_->Stop();
5541}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005542
5543TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5544 // Configure internal source factory and setup test again.
5545 encoder_factory_.SetHasInternalSource(true);
5546 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005548 DataRate::BitsPerSec(kTargetBitrateBps),
5549 DataRate::BitsPerSec(kTargetBitrateBps),
5550 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005551
5552 int64_t timestamp = 1;
5553 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005554 image.SetEncodedData(
5555 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005556 image.capture_time_ms_ = ++timestamp;
5557 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5558 const int64_t kEncodeFinishDelayMs = 10;
5559 image.timing_.encode_start_ms = timestamp;
5560 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5561 fake_encoder_.InjectEncodedImage(image);
5562 // Wait for frame without incrementing clock.
5563 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5564 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5565 // capture timestamp should be kEncodeFinishDelayMs in the past.
5566 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005567 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005568
5569 video_stream_encoder_->Stop();
5570}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005571
5572TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005573 // SPS contains VUI with restrictions on the maximum number of reordered
5574 // pictures, there is no need to rewrite the bitstream to enable faster
5575 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005576 ResetEncoder("H264", 1, 1, 1, false);
5577
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5579 DataRate::BitsPerSec(kTargetBitrateBps),
5580 DataRate::BitsPerSec(kTargetBitrateBps),
5581 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5582 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005583
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005584 fake_encoder_.SetEncodedImageData(
5585 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005586
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005587 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5588 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005589
5590 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5591 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005592
5593 video_stream_encoder_->Stop();
5594}
5595
5596TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005597 // SPS does not contain VUI, the bitstream is will be rewritten with added
5598 // VUI with restrictions on the maximum number of reordered pictures to
5599 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005600 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5601 0x00, 0x00, 0x03, 0x03, 0xF4,
5602 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005603 ResetEncoder("H264", 1, 1, 1, false);
5604
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5606 DataRate::BitsPerSec(kTargetBitrateBps),
5607 DataRate::BitsPerSec(kTargetBitrateBps),
5608 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5609 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005610
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005611 fake_encoder_.SetEncodedImageData(
5612 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005613
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005614 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5615 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005616
5617 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5618 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005619
5620 video_stream_encoder_->Stop();
5621}
5622
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005623TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5624 const int kFrameWidth = 1280;
5625 const int kFrameHeight = 720;
5626 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5627
Henrik Boström381d1092020-05-12 18:49:07 +02005628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005629 DataRate::BitsPerSec(kTargetBitrateBps),
5630 DataRate::BitsPerSec(kTargetBitrateBps),
5631 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005632 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5633
5634 // Insert a first video frame. It should be dropped because of downscale in
5635 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005636 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005637 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5638 frame.set_rotation(kVideoRotation_270);
5639 video_source_.IncomingCapturedFrame(frame);
5640
5641 ExpectDroppedFrame();
5642
5643 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005644 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005645 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5646 frame.set_rotation(kVideoRotation_90);
5647 video_source_.IncomingCapturedFrame(frame);
5648
5649 WaitForEncodedFrame(timestamp_ms);
5650 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5651
5652 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005653 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005654 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5655 frame.set_rotation(kVideoRotation_180);
5656 video_source_.IncomingCapturedFrame(frame);
5657
5658 WaitForEncodedFrame(timestamp_ms);
5659 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5660
5661 video_stream_encoder_->Stop();
5662}
5663
Erik Språng5056af02019-09-02 15:53:11 +02005664TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5665 const int kFrameWidth = 320;
5666 const int kFrameHeight = 180;
5667
5668 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005669 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005670 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5671 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5672 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005673 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005674 /*rtt_ms=*/0,
5675 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005676
5677 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005678 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02005679 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5680 frame.set_rotation(kVideoRotation_270);
5681 video_source_.IncomingCapturedFrame(frame);
5682 WaitForEncodedFrame(timestamp_ms);
5683
5684 // Set a target rate below the minimum allowed by the codec settings.
5685 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005686 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5687 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005688 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005689 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005690 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005691 /*link_allocation=*/target_rate,
5692 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005693 /*rtt_ms=*/0,
5694 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005695 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5696
5697 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5698 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5699 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005700 DataRate allocation_sum =
5701 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005702 EXPECT_EQ(min_rate, allocation_sum);
5703 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5704
5705 video_stream_encoder_->Stop();
5706}
5707
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005708TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005710 DataRate::BitsPerSec(kTargetBitrateBps),
5711 DataRate::BitsPerSec(kTargetBitrateBps),
5712 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005713 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005714 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005715 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5716 WaitForEncodedFrame(1);
5717
5718 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5719 ASSERT_TRUE(prev_rate_settings.has_value());
5720 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5721 kDefaultFramerate);
5722
5723 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5724 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5725 timestamp_ms += 1000 / kDefaultFramerate;
5726 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5727 WaitForEncodedFrame(timestamp_ms);
5728 }
5729 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5730 kDefaultFramerate);
5731 // Capture larger frame to trigger a reconfigure.
5732 codec_height_ *= 2;
5733 codec_width_ *= 2;
5734 timestamp_ms += 1000 / kDefaultFramerate;
5735 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5736 WaitForEncodedFrame(timestamp_ms);
5737
5738 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5739 auto current_rate_settings =
5740 fake_encoder_.GetAndResetLastRateControlSettings();
5741 // Ensure we have actually reconfigured twice
5742 // The rate settings should have been set again even though
5743 // they haven't changed.
5744 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005745 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005746
5747 video_stream_encoder_->Stop();
5748}
5749
philipeld9cc8c02019-09-16 14:53:40 +02005750struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005751 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5752 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5753 MOCK_METHOD(void,
5754 RequestEncoderSwitch,
5755 (const webrtc::SdpVideoFormat& format),
5756 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005757};
5758
5759TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5760 constexpr int kDontCare = 100;
5761
5762 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5763 video_send_config_.encoder_settings.encoder_switch_request_callback =
5764 &switch_callback;
5765 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5766 encoder_config.codec_type = kVideoCodecVP8;
5767 webrtc::test::ScopedFieldTrials field_trial(
5768 "WebRTC-NetworkCondition-EncoderSwitch/"
5769 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5770 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5771
5772 // Reset encoder for new configuration to take effect.
5773 ConfigureEncoder(std::move(encoder_config));
5774
5775 // Send one frame to trigger ReconfigureEncoder.
5776 video_source_.IncomingCapturedFrame(
5777 CreateFrame(kDontCare, kDontCare, kDontCare));
5778
5779 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005780 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5781 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005782 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005783 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005784
Henrik Boström381d1092020-05-12 18:49:07 +02005785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005786 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5787 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5788 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005789 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005790 /*rtt_ms=*/0,
5791 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02005792 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02005793
5794 video_stream_encoder_->Stop();
5795}
5796
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005797TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5798 constexpr int kDontCare = 100;
5799
5800 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5801 video_send_config_.encoder_settings.encoder_switch_request_callback =
5802 &switch_callback;
5803 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5804 encoder_config.codec_type = kVideoCodecVP8;
5805 webrtc::test::ScopedFieldTrials field_trial(
5806 "WebRTC-NetworkCondition-EncoderSwitch/"
5807 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5808 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5809
5810 // Reset encoder for new configuration to take effect.
5811 ConfigureEncoder(std::move(encoder_config));
5812
5813 // Send one frame to trigger ReconfigureEncoder.
5814 video_source_.IncomingCapturedFrame(
5815 CreateFrame(kDontCare, kDontCare, kDontCare));
5816
5817 using Config = EncoderSwitchRequestCallback::Config;
5818 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5819 .Times(0);
5820
Henrik Boström381d1092020-05-12 18:49:07 +02005821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005822 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5823 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5824 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5825 /*fraction_lost=*/0,
5826 /*rtt_ms=*/0,
5827 /*cwnd_reduce_ratio=*/0);
5828
5829 video_stream_encoder_->Stop();
5830}
5831
philipeld9cc8c02019-09-16 14:53:40 +02005832TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5833 constexpr int kSufficientBitrateToNotDrop = 1000;
5834 constexpr int kHighRes = 500;
5835 constexpr int kLowRes = 100;
5836
5837 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5838 video_send_config_.encoder_settings.encoder_switch_request_callback =
5839 &switch_callback;
5840 webrtc::test::ScopedFieldTrials field_trial(
5841 "WebRTC-NetworkCondition-EncoderSwitch/"
5842 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5843 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5844 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5845 encoder_config.codec_type = kVideoCodecH264;
5846
5847 // Reset encoder for new configuration to take effect.
5848 ConfigureEncoder(std::move(encoder_config));
5849
5850 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5851 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5852 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005854 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5855 /*stable_target_bitrate=*/
5856 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5857 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005858 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005859 /*rtt_ms=*/0,
5860 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005861
5862 // Send one frame to trigger ReconfigureEncoder.
5863 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5864 WaitForEncodedFrame(1);
5865
5866 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005867 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5868 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005869 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005870 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005871
5872 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5873 WaitForEncodedFrame(2);
5874
5875 video_stream_encoder_->Stop();
5876}
5877
philipel9b058032020-02-10 11:30:00 +01005878TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5879 constexpr int kDontCare = 100;
5880 StrictMock<MockEncoderSelector> encoder_selector;
5881 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5882 &fake_encoder_, &encoder_selector);
5883 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5884
5885 // Reset encoder for new configuration to take effect.
5886 ConfigureEncoder(video_encoder_config_.Copy());
5887
5888 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5889
5890 video_source_.IncomingCapturedFrame(
5891 CreateFrame(kDontCare, kDontCare, kDontCare));
5892 video_stream_encoder_->Stop();
5893
5894 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5895 // to it's factory, so in order for the encoder instance in the
5896 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5897 // reset the |video_stream_encoder_| here.
5898 video_stream_encoder_.reset();
5899}
5900
5901TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5902 constexpr int kDontCare = 100;
5903
5904 NiceMock<MockEncoderSelector> encoder_selector;
5905 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5906 video_send_config_.encoder_settings.encoder_switch_request_callback =
5907 &switch_callback;
5908 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5909 &fake_encoder_, &encoder_selector);
5910 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5911
5912 // Reset encoder for new configuration to take effect.
5913 ConfigureEncoder(video_encoder_config_.Copy());
5914
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005915 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005916 .WillByDefault(Return(SdpVideoFormat("AV1")));
5917 EXPECT_CALL(switch_callback,
5918 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5919 Field(&SdpVideoFormat::name, "AV1"))));
5920
Henrik Boström381d1092020-05-12 18:49:07 +02005921 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005922 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5923 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5924 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005925 /*fraction_lost=*/0,
5926 /*rtt_ms=*/0,
5927 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02005928 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01005929
5930 video_stream_encoder_->Stop();
5931}
5932
5933TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5934 constexpr int kSufficientBitrateToNotDrop = 1000;
5935 constexpr int kDontCare = 100;
5936
5937 NiceMock<MockVideoEncoder> video_encoder;
5938 NiceMock<MockEncoderSelector> encoder_selector;
5939 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5940 video_send_config_.encoder_settings.encoder_switch_request_callback =
5941 &switch_callback;
5942 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5943 &video_encoder, &encoder_selector);
5944 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5945
5946 // Reset encoder for new configuration to take effect.
5947 ConfigureEncoder(video_encoder_config_.Copy());
5948
5949 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5950 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5951 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005952 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005953 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5954 /*stable_target_bitrate=*/
5955 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5956 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005957 /*fraction_lost=*/0,
5958 /*rtt_ms=*/0,
5959 /*cwnd_reduce_ratio=*/0);
5960
5961 ON_CALL(video_encoder, Encode(_, _))
5962 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5963 ON_CALL(encoder_selector, OnEncoderBroken())
5964 .WillByDefault(Return(SdpVideoFormat("AV2")));
5965
5966 rtc::Event encode_attempted;
5967 EXPECT_CALL(switch_callback,
5968 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5969 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5970 EXPECT_EQ(format.name, "AV2");
5971 encode_attempted.Set();
5972 });
5973
5974 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5975 encode_attempted.Wait(3000);
5976
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02005977 AdvanceTime(TimeDelta::Millis(0));
5978
philipel9b058032020-02-10 11:30:00 +01005979 video_stream_encoder_->Stop();
5980
5981 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5982 // to it's factory, so in order for the encoder instance in the
5983 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5984 // reset the |video_stream_encoder_| here.
5985 video_stream_encoder_.reset();
5986}
5987
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005988TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005989 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005990 const int kFrameWidth = 320;
5991 const int kFrameHeight = 180;
5992
5993 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005994 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005996 /*target_bitrate=*/rate,
5997 /*stable_target_bitrate=*/rate,
5998 /*link_allocation=*/rate,
5999 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006000 /*rtt_ms=*/0,
6001 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006002
6003 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006004 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006005 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6006 frame.set_rotation(kVideoRotation_270);
6007 video_source_.IncomingCapturedFrame(frame);
6008 WaitForEncodedFrame(timestamp_ms);
6009 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6010
6011 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006012 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02006013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006014 /*target_bitrate=*/new_stable_rate,
6015 /*stable_target_bitrate=*/new_stable_rate,
6016 /*link_allocation=*/rate,
6017 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006018 /*rtt_ms=*/0,
6019 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006020 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6021 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
6022 video_stream_encoder_->Stop();
6023}
6024
6025TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006026 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006027 const int kFrameWidth = 320;
6028 const int kFrameHeight = 180;
6029
6030 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006031 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02006032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006033 /*target_bitrate=*/rate,
6034 /*stable_target_bitrate=*/rate,
6035 /*link_allocation=*/rate,
6036 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006037 /*rtt_ms=*/0,
6038 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006039
6040 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006041 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006042 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6043 frame.set_rotation(kVideoRotation_270);
6044 video_source_.IncomingCapturedFrame(frame);
6045 WaitForEncodedFrame(timestamp_ms);
6046 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6047
6048 // Set a higher target rate without changing the link_allocation. Should not
6049 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006050 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02006051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006052 /*target_bitrate=*/rate,
6053 /*stable_target_bitrate=*/new_stable_rate,
6054 /*link_allocation=*/rate,
6055 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006056 /*rtt_ms=*/0,
6057 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006058 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6059 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
6060 video_stream_encoder_->Stop();
6061}
6062
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006063TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
6064 test::ScopedFieldTrials field_trials(
6065 "WebRTC-AutomaticAnimationDetectionScreenshare/"
6066 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
6067 const int kFramerateFps = 30;
6068 const int kWidth = 1920;
6069 const int kHeight = 1080;
6070 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
6071 // Works on screenshare mode.
6072 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
6073 // We rely on the automatic resolution adaptation, but we handle framerate
6074 // adaptation manually by mocking the stats proxy.
6075 video_source_.set_adaptation_enabled(true);
6076
6077 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02006078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006079 DataRate::BitsPerSec(kTargetBitrateBps),
6080 DataRate::BitsPerSec(kTargetBitrateBps),
6081 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006082 video_stream_encoder_->SetSource(&video_source_,
6083 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006084 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006085
6086 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6087 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
6088
6089 // Pass enough frames with the full update to trigger animation detection.
6090 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006091 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006092 frame.set_ntp_time_ms(timestamp_ms);
6093 frame.set_timestamp_us(timestamp_ms * 1000);
6094 video_source_.IncomingCapturedFrame(frame);
6095 WaitForEncodedFrame(timestamp_ms);
6096 }
6097
6098 // Resolution should be limited.
6099 rtc::VideoSinkWants expected;
6100 expected.max_framerate_fps = kFramerateFps;
6101 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006102 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006103
6104 // Pass one frame with no known update.
6105 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006106 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006107 frame.set_ntp_time_ms(timestamp_ms);
6108 frame.set_timestamp_us(timestamp_ms * 1000);
6109 frame.clear_update_rect();
6110
6111 video_source_.IncomingCapturedFrame(frame);
6112 WaitForEncodedFrame(timestamp_ms);
6113
6114 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006115 EXPECT_THAT(video_source_.sink_wants(),
6116 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006117
6118 video_stream_encoder_->Stop();
6119}
6120
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02006121TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
6122 const int kWidth = 720; // 540p adapted down.
6123 const int kHeight = 405;
6124 const int kNumFrames = 3;
6125 // Works on screenshare mode.
6126 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
6127 /*num_spatial_layers=*/2, /*screenshare=*/true);
6128
6129 video_source_.set_adaptation_enabled(true);
6130
6131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6132 DataRate::BitsPerSec(kTargetBitrateBps),
6133 DataRate::BitsPerSec(kTargetBitrateBps),
6134 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6135
6136 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6137
6138 // Pass enough frames with the full update to trigger animation detection.
6139 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006140 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02006141 frame.set_ntp_time_ms(timestamp_ms);
6142 frame.set_timestamp_us(timestamp_ms * 1000);
6143 video_source_.IncomingCapturedFrame(frame);
6144 WaitForEncodedFrame(timestamp_ms);
6145 }
6146
6147 video_stream_encoder_->Stop();
6148}
6149
perkj26091b12016-09-01 01:17:40 -07006150} // namespace webrtc