blob: 0cacbbea2f1fce23359fbfd07154550dc1577732 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020029#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010030#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020031#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070032#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020034#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020035#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010036#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020038#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080040#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020041#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010042#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020043#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020044#include "system_wrappers/include/sleep.h"
45#include "test/encoder_settings.h"
46#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020047#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010048#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020049#include "test/gmock.h"
50#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020051#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020052#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070053
54namespace webrtc {
55
sprang57c2fff2017-01-16 06:24:02 -080056using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020057using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020058using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020060using ::testing::Ge;
61using ::testing::Gt;
62using ::testing::Le;
63using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010064using ::testing::Matcher;
65using ::testing::NiceMock;
66using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020067using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080068
perkj803d97f2016-11-01 11:45:46 -070069namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020070const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010071const int kQpLow = 1;
72const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020073const int kMinFramerateFps = 2;
74const int kMinBalancedFramerateFps = 7;
75const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080076const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010077const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020078const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010079const uint32_t kSimulcastTargetBitrateBps = 3150000;
80const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080081const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070082const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020083const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020084const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020085const VideoEncoder::ResolutionBitrateLimits
86 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
87const VideoEncoder::ResolutionBitrateLimits
88 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080089
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020090uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
91 0x00, 0x00, 0x03, 0x03, 0xF4,
92 0x05, 0x03, 0xC7, 0xE0, 0x1B,
93 0x41, 0x10, 0x8D, 0x00};
94
perkj803d97f2016-11-01 11:45:46 -070095class TestBuffer : public webrtc::I420Buffer {
96 public:
97 TestBuffer(rtc::Event* event, int width, int height)
98 : I420Buffer(width, height), event_(event) {}
99
100 private:
101 friend class rtc::RefCountedObject<TestBuffer>;
102 ~TestBuffer() override {
103 if (event_)
104 event_->Set();
105 }
106 rtc::Event* const event_;
107};
108
Noah Richards51db4212019-06-12 06:59:12 -0700109// A fake native buffer that can't be converted to I420.
110class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
111 public:
112 FakeNativeBuffer(rtc::Event* event, int width, int height)
113 : event_(event), width_(width), height_(height) {}
114 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
115 int width() const override { return width_; }
116 int height() const override { return height_; }
117 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
118 return nullptr;
119 }
120
121 private:
122 friend class rtc::RefCountedObject<FakeNativeBuffer>;
123 ~FakeNativeBuffer() override {
124 if (event_)
125 event_->Set();
126 }
127 rtc::Event* const event_;
128 const int width_;
129 const int height_;
130};
131
Niels Möller7dc26b72017-12-06 10:27:48 +0100132class CpuOveruseDetectorProxy : public OveruseFrameDetector {
133 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200134 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
135 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200136 last_target_framerate_fps_(-1),
137 framerate_updated_event_(true /* manual_reset */,
138 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100139 virtual ~CpuOveruseDetectorProxy() {}
140
141 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200142 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100143 last_target_framerate_fps_ = framerate_fps;
144 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200145 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100146 }
147
148 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200149 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100150 return last_target_framerate_fps_;
151 }
152
Niels Möller4db138e2018-04-19 09:04:13 +0200153 CpuOveruseOptions GetOptions() { return options_; }
154
Henrik Boström381d1092020-05-12 18:49:07 +0200155 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
156
Niels Möller7dc26b72017-12-06 10:27:48 +0100157 private:
Markus Handella3765182020-07-08 13:13:32 +0200158 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100159 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200160 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100161};
162
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200163class FakeVideoSourceRestrictionsListener
164 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200165 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200166 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200167 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200168 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200169 RTC_DCHECK(was_restrictions_updated_);
170 }
171
172 rtc::Event* restrictions_updated_event() {
173 return &restrictions_updated_event_;
174 }
175
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200176 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200177 void OnVideoSourceRestrictionsUpdated(
178 VideoSourceRestrictions restrictions,
179 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200180 rtc::scoped_refptr<Resource> reason,
181 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200182 was_restrictions_updated_ = true;
183 restrictions_updated_event_.Set();
184 }
185
186 private:
187 bool was_restrictions_updated_;
188 rtc::Event restrictions_updated_event_;
189};
190
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200191auto WantsFps(Matcher<int> fps_matcher) {
192 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
193 fps_matcher);
194}
195
196auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
197 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
198 AllOf(max_pixel_matcher, Gt(0)));
199}
200
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200201auto ResolutionMax() {
202 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200203 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200204 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
205 Eq(absl::nullopt)));
206}
207
208auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200209 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200210}
211
212auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200213 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200214}
215
216auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200217 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200218}
219
220auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200221 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200222}
223
224auto FpsMaxResolutionMax() {
225 return AllOf(FpsMax(), ResolutionMax());
226}
227
228auto UnlimitedSinkWants() {
229 return AllOf(FpsUnlimited(), ResolutionMax());
230}
231
232auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
233 Matcher<int> fps_range_matcher;
234
235 if (last_frame_pixels <= 320 * 240) {
236 fps_range_matcher = AllOf(Ge(7), Le(10));
237 } else if (last_frame_pixels <= 480 * 270) {
238 fps_range_matcher = AllOf(Ge(10), Le(15));
239 } else if (last_frame_pixels <= 640 * 480) {
240 fps_range_matcher = Ge(15);
241 } else {
242 fps_range_matcher = Eq(kDefaultFramerate);
243 }
244 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
245 fps_range_matcher);
246}
247
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
249 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
250 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
251}
252
253auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
254 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
255}
256
257auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
258 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
259}
260
261auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
262 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
263 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
264}
265
266auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
267 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
268 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
269}
270
271auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
272 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
273 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
274}
275
276auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
277 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
278 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
279}
280
mflodmancc3d4422017-08-03 08:27:51 -0700281class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700282 public:
Niels Möller213618e2018-07-24 09:29:58 +0200283 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200284 const VideoStreamEncoderSettings& settings,
285 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100286 : VideoStreamEncoder(Clock::GetRealTimeClock(),
287 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200288 stats_proxy,
289 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200290 std::unique_ptr<OveruseFrameDetector>(
291 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100292 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100293 task_queue_factory),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200294 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200295 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200296 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200297 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200298 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200299 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200300 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100301 }
perkj803d97f2016-11-01 11:45:46 -0700302
Henrik Boström381d1092020-05-12 18:49:07 +0200303 void SetSourceAndWaitForRestrictionsUpdated(
304 rtc::VideoSourceInterface<VideoFrame>* source,
305 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200306 FakeVideoSourceRestrictionsListener listener;
307 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200308 SetSource(source, degradation_preference);
309 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200310 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200311 }
312
313 void SetSourceAndWaitForFramerateUpdated(
314 rtc::VideoSourceInterface<VideoFrame>* source,
315 const DegradationPreference& degradation_preference) {
316 overuse_detector_proxy_->framerate_updated_event()->Reset();
317 SetSource(source, degradation_preference);
318 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
319 }
320
321 void OnBitrateUpdatedAndWaitForManagedResources(
322 DataRate target_bitrate,
323 DataRate stable_target_bitrate,
324 DataRate link_allocation,
325 uint8_t fraction_lost,
326 int64_t round_trip_time_ms,
327 double cwnd_reduce_ratio) {
328 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
329 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
330 // Bitrate is updated on the encoder queue.
331 WaitUntilTaskQueueIsIdle();
332 // Give the managed resources time to react to the new bitrate.
333 // TODO(hbos): Can we await an appropriate event instead?
334 WaitUntilAdaptationTaskQueueIsIdle();
335 }
336
337 void WaitUntilAdaptationTaskQueueIsIdle() {
338 rtc::Event event;
339 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
340 ASSERT_TRUE(event.Wait(5000));
341 }
342
kthelgason2fc52542017-03-03 00:24:41 -0800343 // This is used as a synchronisation mechanism, to make sure that the
344 // encoder queue is not blocked before we start sending it frames.
345 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100346 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200347 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800348 ASSERT_TRUE(event.Wait(5000));
349 }
350
Henrik Boström91aa7322020-04-28 12:24:33 +0200351 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200352 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200353 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200354 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200355 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200356 event.Set();
357 });
358 ASSERT_TRUE(event.Wait(5000));
359 }
360 void TriggerCpuUnderuse() {
361 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200362 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200363 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200364 event.Set();
365 });
366 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200367 }
kthelgason876222f2016-11-29 01:44:11 -0800368
Henrik Boström91aa7322020-04-28 12:24:33 +0200369 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200370 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200371 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200372 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200374 event.Set();
375 });
376 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200377 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200378 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200379 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200380 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200381 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 event.Set();
383 });
384 ASSERT_TRUE(event.Wait(5000));
385 }
386
Niels Möller7dc26b72017-12-06 10:27:48 +0100387 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200388 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
389 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200390 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700391};
392
asapersson5f7226f2016-11-25 04:37:00 -0800393class VideoStreamFactory
394 : public VideoEncoderConfig::VideoStreamFactoryInterface {
395 public:
sprangfda496a2017-06-15 04:21:07 -0700396 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
397 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800398 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700399 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800400 }
401
402 private:
403 std::vector<VideoStream> CreateEncoderStreams(
404 int width,
405 int height,
406 const VideoEncoderConfig& encoder_config) override {
407 std::vector<VideoStream> streams =
408 test::CreateVideoStreams(width, height, encoder_config);
409 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100410 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700411 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800412 }
413 return streams;
414 }
sprangfda496a2017-06-15 04:21:07 -0700415
asapersson5f7226f2016-11-25 04:37:00 -0800416 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700417 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800418};
419
Noah Richards51db4212019-06-12 06:59:12 -0700420// Simulates simulcast behavior and makes highest stream resolutions divisible
421// by 4.
422class CroppingVideoStreamFactory
423 : public VideoEncoderConfig::VideoStreamFactoryInterface {
424 public:
425 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
426 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
427 EXPECT_GT(num_temporal_layers, 0u);
428 EXPECT_GT(framerate, 0);
429 }
430
431 private:
432 std::vector<VideoStream> CreateEncoderStreams(
433 int width,
434 int height,
435 const VideoEncoderConfig& encoder_config) override {
436 std::vector<VideoStream> streams = test::CreateVideoStreams(
437 width - width % 4, height - height % 4, encoder_config);
438 for (VideoStream& stream : streams) {
439 stream.num_temporal_layers = num_temporal_layers_;
440 stream.max_framerate = framerate_;
441 }
442 return streams;
443 }
444
445 const size_t num_temporal_layers_;
446 const int framerate_;
447};
448
sprangb1ca0732017-02-01 08:38:12 -0800449class AdaptingFrameForwarder : public test::FrameForwarder {
450 public:
451 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700452 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800453
454 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200455 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800456 adaptation_enabled_ = enabled;
457 }
458
asaperssonfab67072017-04-04 05:51:49 -0700459 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200460 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800461 return adaptation_enabled_;
462 }
463
asapersson09f05612017-05-15 23:40:18 -0700464 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200465 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700466 return last_wants_;
467 }
468
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200469 absl::optional<int> last_sent_width() const { return last_width_; }
470 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800471
sprangb1ca0732017-02-01 08:38:12 -0800472 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
473 int cropped_width = 0;
474 int cropped_height = 0;
475 int out_width = 0;
476 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700477 if (adaption_enabled()) {
478 if (adapter_.AdaptFrameResolution(
479 video_frame.width(), video_frame.height(),
480 video_frame.timestamp_us() * 1000, &cropped_width,
481 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100482 VideoFrame adapted_frame =
483 VideoFrame::Builder()
484 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
485 nullptr, out_width, out_height))
486 .set_timestamp_rtp(99)
487 .set_timestamp_ms(99)
488 .set_rotation(kVideoRotation_0)
489 .build();
sprangc5d62e22017-04-02 23:53:04 -0700490 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100491 if (video_frame.has_update_rect()) {
492 adapted_frame.set_update_rect(
493 video_frame.update_rect().ScaleWithFrame(
494 video_frame.width(), video_frame.height(), 0, 0,
495 video_frame.width(), video_frame.height(), out_width,
496 out_height));
497 }
sprangc5d62e22017-04-02 23:53:04 -0700498 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800499 last_width_.emplace(adapted_frame.width());
500 last_height_.emplace(adapted_frame.height());
501 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200502 last_width_ = absl::nullopt;
503 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700504 }
sprangb1ca0732017-02-01 08:38:12 -0800505 } else {
506 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800507 last_width_.emplace(video_frame.width());
508 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800509 }
510 }
511
512 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
513 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200514 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200515 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100516 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200517 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800518 }
sprangb1ca0732017-02-01 08:38:12 -0800519 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200520 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
521 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200522 absl::optional<int> last_width_;
523 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800524};
sprangc5d62e22017-04-02 23:53:04 -0700525
Niels Möller213618e2018-07-24 09:29:58 +0200526// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700527class MockableSendStatisticsProxy : public SendStatisticsProxy {
528 public:
529 MockableSendStatisticsProxy(Clock* clock,
530 const VideoSendStream::Config& config,
531 VideoEncoderConfig::ContentType content_type)
532 : SendStatisticsProxy(clock, config, content_type) {}
533
534 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200535 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700536 if (mock_stats_)
537 return *mock_stats_;
538 return SendStatisticsProxy::GetStats();
539 }
540
Niels Möller213618e2018-07-24 09:29:58 +0200541 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200542 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200543 if (mock_stats_)
544 return mock_stats_->input_frame_rate;
545 return SendStatisticsProxy::GetInputFrameRate();
546 }
sprangc5d62e22017-04-02 23:53:04 -0700547 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200548 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700549 mock_stats_.emplace(stats);
550 }
551
552 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200553 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700554 mock_stats_.reset();
555 }
556
557 private:
Markus Handella3765182020-07-08 13:13:32 +0200558 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200559 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700560};
561
sprang4847ae62017-06-27 07:06:52 -0700562class MockBitrateObserver : public VideoBitrateAllocationObserver {
563 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200564 MOCK_METHOD(void,
565 OnBitrateAllocationUpdated,
566 (const VideoBitrateAllocation&),
567 (override));
sprang4847ae62017-06-27 07:06:52 -0700568};
569
philipel9b058032020-02-10 11:30:00 +0100570class MockEncoderSelector
571 : public VideoEncoderFactory::EncoderSelectorInterface {
572 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200573 MOCK_METHOD(void,
574 OnCurrentEncoder,
575 (const SdpVideoFormat& format),
576 (override));
577 MOCK_METHOD(absl::optional<SdpVideoFormat>,
578 OnAvailableBitrate,
579 (const DataRate& rate),
580 (override));
581 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100582};
583
perkj803d97f2016-11-01 11:45:46 -0700584} // namespace
585
mflodmancc3d4422017-08-03 08:27:51 -0700586class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700587 public:
588 static const int kDefaultTimeoutMs = 30 * 1000;
589
mflodmancc3d4422017-08-03 08:27:51 -0700590 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700591 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700592 codec_width_(320),
593 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200594 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200595 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700596 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200597 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700598 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700599 Clock::GetRealTimeClock(),
600 video_send_config_,
601 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700602 sink_(&fake_encoder_) {}
603
604 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700605 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700606 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200607 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800608 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200609 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200610 video_send_config_.rtp.payload_name = "FAKE";
611 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700612
Per512ecb32016-09-23 15:52:06 +0200613 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200614 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700615 video_encoder_config.video_stream_factory =
616 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100617 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700618
619 // Framerate limit is specified by the VideoStreamFactory.
620 std::vector<VideoStream> streams =
621 video_encoder_config.video_stream_factory->CreateEncoderStreams(
622 codec_width_, codec_height_, video_encoder_config);
623 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100624 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700625
Niels Möllerf1338562018-04-26 09:51:47 +0200626 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800627 }
628
Niels Möllerf1338562018-04-26 09:51:47 +0200629 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700630 if (video_stream_encoder_)
631 video_stream_encoder_->Stop();
632 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200633 stats_proxy_.get(), video_send_config_.encoder_settings,
634 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700635 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
636 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700637 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700638 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
639 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200640 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700641 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800642 }
643
644 void ResetEncoder(const std::string& payload_name,
645 size_t num_streams,
646 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700647 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700648 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200649 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800650
651 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200652 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800653 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100654 video_encoder_config.max_bitrate_bps =
655 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800656 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700657 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
658 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700659 video_encoder_config.content_type =
660 screenshare ? VideoEncoderConfig::ContentType::kScreen
661 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700662 if (payload_name == "VP9") {
663 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
664 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200665 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700666 video_encoder_config.encoder_specific_settings =
667 new rtc::RefCountedObject<
668 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
669 }
Niels Möllerf1338562018-04-26 09:51:47 +0200670 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700671 }
672
sprang57c2fff2017-01-16 06:24:02 -0800673 VideoFrame CreateFrame(int64_t ntp_time_ms,
674 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100675 VideoFrame frame =
676 VideoFrame::Builder()
677 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
678 destruction_event, codec_width_, codec_height_))
679 .set_timestamp_rtp(99)
680 .set_timestamp_ms(99)
681 .set_rotation(kVideoRotation_0)
682 .build();
sprang57c2fff2017-01-16 06:24:02 -0800683 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700684 return frame;
685 }
686
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100687 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
688 rtc::Event* destruction_event,
689 int offset_x) const {
690 VideoFrame frame =
691 VideoFrame::Builder()
692 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
693 destruction_event, codec_width_, codec_height_))
694 .set_timestamp_rtp(99)
695 .set_timestamp_ms(99)
696 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100697 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100698 .build();
699 frame.set_ntp_time_ms(ntp_time_ms);
700 return frame;
701 }
702
sprang57c2fff2017-01-16 06:24:02 -0800703 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100704 VideoFrame frame =
705 VideoFrame::Builder()
706 .set_video_frame_buffer(
707 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
708 .set_timestamp_rtp(99)
709 .set_timestamp_ms(99)
710 .set_rotation(kVideoRotation_0)
711 .build();
sprang57c2fff2017-01-16 06:24:02 -0800712 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700713 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700714 return frame;
715 }
716
Noah Richards51db4212019-06-12 06:59:12 -0700717 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
718 rtc::Event* destruction_event,
719 int width,
720 int height) const {
721 VideoFrame frame =
722 VideoFrame::Builder()
723 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
724 destruction_event, width, height))
725 .set_timestamp_rtp(99)
726 .set_timestamp_ms(99)
727 .set_rotation(kVideoRotation_0)
728 .build();
729 frame.set_ntp_time_ms(ntp_time_ms);
730 return frame;
731 }
732
733 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
734 rtc::Event* destruction_event) const {
735 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
736 codec_height_);
737 }
738
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100739 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
740 MockBitrateObserver bitrate_observer;
741 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
742
743 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
744 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100746 DataRate::BitsPerSec(kTargetBitrateBps),
747 DataRate::BitsPerSec(kTargetBitrateBps),
748 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100749
750 video_source_.IncomingCapturedFrame(
751 CreateFrame(1, codec_width_, codec_height_));
752 WaitForEncodedFrame(1);
753 }
754
sprang4847ae62017-06-27 07:06:52 -0700755 void WaitForEncodedFrame(int64_t expected_ntp_time) {
756 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100757 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700758 }
759
760 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
761 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100762 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700763 return ok;
764 }
765
766 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
767 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100768 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700769 }
770
771 void ExpectDroppedFrame() {
772 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100773 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700774 }
775
776 bool WaitForFrame(int64_t timeout_ms) {
777 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100778 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700779 return ok;
780 }
781
perkj26091b12016-09-01 01:17:40 -0700782 class TestEncoder : public test::FakeEncoder {
783 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100784 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700785
asaperssonfab67072017-04-04 05:51:49 -0700786 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200787 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700788 return config_;
789 }
790
791 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200792 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700793 block_next_encode_ = true;
794 }
795
Erik Språngaed30702018-11-05 12:57:17 +0100796 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200797 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100798 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100799 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100800 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100801 info.scaling_settings = VideoEncoder::ScalingSettings(
802 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100803 }
804 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100805 for (int i = 0; i < kMaxSpatialLayers; ++i) {
806 if (temporal_layers_supported_[i]) {
807 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
808 info.fps_allocation[i].resize(num_layers);
809 }
810 }
Erik Språngaed30702018-11-05 12:57:17 +0100811 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200812
813 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100814 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100815 return info;
kthelgason876222f2016-11-29 01:44:11 -0800816 }
817
Erik Språngb7cb7b52019-02-26 15:52:33 +0100818 int32_t RegisterEncodeCompleteCallback(
819 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200820 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100821 encoded_image_callback_ = callback;
822 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
823 }
824
perkjfa10b552016-10-02 23:45:26 -0700825 void ContinueEncode() { continue_encode_event_.Set(); }
826
827 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
828 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200829 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700830 EXPECT_EQ(timestamp_, timestamp);
831 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
832 }
833
kthelgason2fc52542017-03-03 00:24:41 -0800834 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200835 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800836 quality_scaling_ = b;
837 }
kthelgasonad9010c2017-02-14 00:46:51 -0800838
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100839 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200840 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100841 requested_resolution_alignment_ = requested_resolution_alignment;
842 }
843
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200845 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100846 is_hardware_accelerated_ = is_hardware_accelerated;
847 }
848
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
850 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200851 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100852 temporal_layers_supported_[spatial_idx] = supported;
853 }
854
Sergey Silkin6456e352019-07-08 17:56:40 +0200855 void SetResolutionBitrateLimits(
856 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200857 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200858 resolution_bitrate_limits_ = thresholds;
859 }
860
sprangfe627f32017-03-29 08:24:59 -0700861 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200862 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700863 force_init_encode_failed_ = force_failure;
864 }
865
Niels Möller6bb5ab92019-01-11 11:11:10 +0100866 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200867 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100868 rate_factor_ = rate_factor;
869 }
870
Erik Språngd7329ca2019-02-21 21:19:53 +0100871 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200872 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100873 return last_framerate_;
874 }
875
Erik Språngd7329ca2019-02-21 21:19:53 +0100876 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200877 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100878 return last_update_rect_;
879 }
880
Niels Möller87e2d782019-03-07 10:18:23 +0100881 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200882 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100883 return last_frame_types_;
884 }
885
886 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100887 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100888 keyframe ? VideoFrameType::kVideoFrameKey
889 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100890 {
Markus Handella3765182020-07-08 13:13:32 +0200891 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100892 last_frame_types_ = frame_type;
893 }
Niels Möllerb859b322019-03-07 12:40:01 +0100894 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100895 }
896
Erik Språngb7cb7b52019-02-26 15:52:33 +0100897 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200898 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100899 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
900 }
901
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200902 void SetEncodedImageData(
903 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200904 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200905 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200906 }
907
Erik Språngd7329ca2019-02-21 21:19:53 +0100908 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200909 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100910 expect_null_frame_ = true;
911 }
912
Erik Språng5056af02019-09-02 15:53:11 +0200913 absl::optional<VideoEncoder::RateControlParameters>
914 GetAndResetLastRateControlSettings() {
915 auto settings = last_rate_control_settings_;
916 last_rate_control_settings_.reset();
917 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100918 }
919
Sergey Silkin5ee69672019-07-02 14:18:34 +0200920 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200922 return num_encoder_initializations_;
923 }
924
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200925 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200927 return num_set_rates_;
928 }
929
perkjfa10b552016-10-02 23:45:26 -0700930 private:
perkj26091b12016-09-01 01:17:40 -0700931 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100932 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700933 bool block_encode;
934 {
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 if (expect_null_frame_) {
937 EXPECT_EQ(input_image.timestamp(), 0u);
938 EXPECT_EQ(input_image.width(), 1);
939 last_frame_types_ = *frame_types;
940 expect_null_frame_ = false;
941 } else {
942 EXPECT_GT(input_image.timestamp(), timestamp_);
943 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
944 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
945 }
perkj26091b12016-09-01 01:17:40 -0700946
947 timestamp_ = input_image.timestamp();
948 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700949 last_input_width_ = input_image.width();
950 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700951 block_encode = block_next_encode_;
952 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100953 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100954 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700955 }
Niels Möllerb859b322019-03-07 12:40:01 +0100956 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700957 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700958 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700959 return result;
960 }
961
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200962 std::unique_ptr<RTPFragmentationHeader> EncodeHook(
963 EncodedImage* encoded_image,
964 CodecSpecificInfo* codec_specific) override {
965 {
966 MutexLock lock(&mutex_);
967 codec_specific->codecType = config_.codecType;
968 }
969 MutexLock lock(&local_mutex_);
970 if (encoded_image_data_) {
971 encoded_image->SetEncodedData(encoded_image_data_);
972 if (codec_specific->codecType == kVideoCodecH264) {
973 auto fragmentation = std::make_unique<RTPFragmentationHeader>();
974 fragmentation->VerifyAndAllocateFragmentationHeader(1);
975 fragmentation->fragmentationOffset[0] = 4;
976 fragmentation->fragmentationLength[0] = encoded_image->size() - 4;
977 return fragmentation;
978 }
979 }
980 return nullptr;
981 }
982
sprangfe627f32017-03-29 08:24:59 -0700983 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200984 const Settings& settings) override {
985 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200986
Markus Handella3765182020-07-08 13:13:32 +0200987 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100988 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200989
990 ++num_encoder_initializations_;
991
Erik Språng82fad3d2018-03-21 09:57:23 +0100992 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700993 // Simulate setting up temporal layers, in order to validate the life
994 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100995 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200996 frame_buffer_controller_ =
997 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700998 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100999 if (force_init_encode_failed_) {
1000 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001001 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001002 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001003
Erik Språngb7cb7b52019-02-26 15:52:33 +01001004 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001005 return res;
1006 }
1007
Erik Språngb7cb7b52019-02-26 15:52:33 +01001008 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001009 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001010 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1011 initialized_ = EncoderState::kUninitialized;
1012 return FakeEncoder::Release();
1013 }
1014
Erik Språng16cb8f52019-04-12 13:59:09 +02001015 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001016 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001017 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001018 VideoBitrateAllocation adjusted_rate_allocation;
1019 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1020 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001021 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001022 adjusted_rate_allocation.SetBitrate(
1023 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001024 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001025 rate_factor_));
1026 }
1027 }
1028 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001029 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001030 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001031 RateControlParameters adjusted_paramters = parameters;
1032 adjusted_paramters.bitrate = adjusted_rate_allocation;
1033 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001034 }
1035
Markus Handella3765182020-07-08 13:13:32 +02001036 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001037 enum class EncoderState {
1038 kUninitialized,
1039 kInitializationFailed,
1040 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001041 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1042 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001043 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001044 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1045 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1046 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1047 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1048 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1049 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
1050 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001051 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1052 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001053 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001054 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001055 absl::optional<bool>
1056 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001057 local_mutex_);
1058 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1059 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1060 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001061 absl::optional<VideoEncoder::RateControlParameters>
1062 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001063 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1064 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001065 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001066 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001067 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1068 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001069 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001070 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001071 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001072 RTC_GUARDED_BY(local_mutex_);
1073 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001074 };
1075
mflodmancc3d4422017-08-03 08:27:51 -07001076 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001077 public:
1078 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001079 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001080
perkj26091b12016-09-01 01:17:40 -07001081 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001082 EXPECT_TRUE(
1083 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1084 }
1085
1086 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1087 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001088 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001089 if (!encoded_frame_event_.Wait(timeout_ms))
1090 return false;
perkj26091b12016-09-01 01:17:40 -07001091 {
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001093 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001094 }
1095 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001096 return true;
perkj26091b12016-09-01 01:17:40 -07001097 }
1098
sprangb1ca0732017-02-01 08:38:12 -08001099 void WaitForEncodedFrame(uint32_t expected_width,
1100 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001101 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001102 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001103 }
1104
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001105 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001106 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001107 uint32_t width = 0;
1108 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001109 {
Markus Handella3765182020-07-08 13:13:32 +02001110 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001111 width = last_width_;
1112 height = last_height_;
1113 }
1114 EXPECT_EQ(expected_height, height);
1115 EXPECT_EQ(expected_width, width);
1116 }
1117
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001118 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1119 int width = 0;
1120 int height = 0;
1121 {
Markus Handella3765182020-07-08 13:13:32 +02001122 MutexLock lock(&mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001123 width = last_width_;
1124 height = last_height_;
1125 }
1126 EXPECT_EQ(width % resolution_alignment, 0);
1127 EXPECT_EQ(height % resolution_alignment, 0);
1128 }
1129
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001130 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1131 VideoRotation rotation;
1132 {
Markus Handella3765182020-07-08 13:13:32 +02001133 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001134 rotation = last_rotation_;
1135 }
1136 EXPECT_EQ(expected_rotation, rotation);
1137 }
1138
kthelgason2fc52542017-03-03 00:24:41 -08001139 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001140
sprangc5d62e22017-04-02 23:53:04 -07001141 bool WaitForFrame(int64_t timeout_ms) {
1142 return encoded_frame_event_.Wait(timeout_ms);
1143 }
1144
perkj26091b12016-09-01 01:17:40 -07001145 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001146 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001147 expect_frames_ = false;
1148 }
1149
asaperssonfab67072017-04-04 05:51:49 -07001150 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001151 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001152 return number_of_reconfigurations_;
1153 }
1154
asaperssonfab67072017-04-04 05:51:49 -07001155 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001156 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001157 return min_transmit_bitrate_bps_;
1158 }
1159
Erik Språngd7329ca2019-02-21 21:19:53 +01001160 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001161 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001162 num_expected_layers_ = num_layers;
1163 }
1164
Erik Språngb7cb7b52019-02-26 15:52:33 +01001165 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001166 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001167 return last_capture_time_ms_;
1168 }
1169
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001170 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001171 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001172 return std::move(last_encoded_image_data_);
1173 }
1174
perkj26091b12016-09-01 01:17:40 -07001175 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001176 Result OnEncodedImage(
1177 const EncodedImage& encoded_image,
1178 const CodecSpecificInfo* codec_specific_info,
Danil Chapovalov6d008a82020-07-22 19:49:36 +02001179 const RTPFragmentationHeader* /*fragmentation*/) override {
Markus Handella3765182020-07-08 13:13:32 +02001180 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001181 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001182 last_encoded_image_data_ = std::vector<uint8_t>(
1183 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001184 uint32_t timestamp = encoded_image.Timestamp();
1185 if (last_timestamp_ != timestamp) {
1186 num_received_layers_ = 1;
1187 } else {
1188 ++num_received_layers_;
1189 }
1190 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001191 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001192 last_width_ = encoded_image._encodedWidth;
1193 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001194 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001195 if (num_received_layers_ == num_expected_layers_) {
1196 encoded_frame_event_.Set();
1197 }
sprangb1ca0732017-02-01 08:38:12 -08001198 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001199 }
1200
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001201 void OnEncoderConfigurationChanged(
1202 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001203 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001204 VideoEncoderConfig::ContentType content_type,
1205 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001206 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001207 ++number_of_reconfigurations_;
1208 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1209 }
1210
Markus Handella3765182020-07-08 13:13:32 +02001211 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001212 TestEncoder* test_encoder_;
1213 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001214 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001215 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001216 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001217 uint32_t last_height_ = 0;
1218 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001219 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001220 size_t num_expected_layers_ = 1;
1221 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001222 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001223 int number_of_reconfigurations_ = 0;
1224 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001225 };
1226
Sergey Silkin5ee69672019-07-02 14:18:34 +02001227 class VideoBitrateAllocatorProxyFactory
1228 : public VideoBitrateAllocatorFactory {
1229 public:
1230 VideoBitrateAllocatorProxyFactory()
1231 : bitrate_allocator_factory_(
1232 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1233
1234 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1235 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001236 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001237 codec_config_ = codec;
1238 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1239 }
1240
1241 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001242 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001243 return codec_config_;
1244 }
1245
1246 private:
1247 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1248
Markus Handella3765182020-07-08 13:13:32 +02001249 mutable Mutex mutex_;
1250 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001251 };
1252
perkj26091b12016-09-01 01:17:40 -07001253 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001254 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001255 int codec_width_;
1256 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001257 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001258 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001259 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001260 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001261 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001262 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001263 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001264 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001265 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001266 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001267};
1268
mflodmancc3d4422017-08-03 08:27:51 -07001269TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001270 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001271 DataRate::BitsPerSec(kTargetBitrateBps),
1272 DataRate::BitsPerSec(kTargetBitrateBps),
1273 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001274 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001275 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001276 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001277 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001278 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001279}
1280
mflodmancc3d4422017-08-03 08:27:51 -07001281TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001282 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001283 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001284 // The encoder will cache up to one frame for a short duration. Adding two
1285 // frames means that the first frame will be dropped and the second frame will
1286 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001287 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001288 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001289 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001290
Henrik Boström381d1092020-05-12 18:49:07 +02001291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001292 DataRate::BitsPerSec(kTargetBitrateBps),
1293 DataRate::BitsPerSec(kTargetBitrateBps),
1294 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001295
Sebastian Janssona3177052018-04-10 13:05:49 +02001296 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001297 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001298 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1299
1300 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001301 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001302}
1303
mflodmancc3d4422017-08-03 08:27:51 -07001304TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001306 DataRate::BitsPerSec(kTargetBitrateBps),
1307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001309 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001310 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001311
Henrik Boström381d1092020-05-12 18:49:07 +02001312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1313 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1314 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001315 // The encoder will cache up to one frame for a short duration. Adding two
1316 // frames means that the first frame will be dropped and the second frame will
1317 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001318 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001319 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001320
Henrik Boström381d1092020-05-12 18:49:07 +02001321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001322 DataRate::BitsPerSec(kTargetBitrateBps),
1323 DataRate::BitsPerSec(kTargetBitrateBps),
1324 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001325 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001326 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1327 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001328 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001329}
1330
mflodmancc3d4422017-08-03 08:27:51 -07001331TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001333 DataRate::BitsPerSec(kTargetBitrateBps),
1334 DataRate::BitsPerSec(kTargetBitrateBps),
1335 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001336 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001337 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001338
1339 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001340 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001341
perkja49cbd32016-09-16 07:53:41 -07001342 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001343 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001344 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001345}
1346
mflodmancc3d4422017-08-03 08:27:51 -07001347TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001349 DataRate::BitsPerSec(kTargetBitrateBps),
1350 DataRate::BitsPerSec(kTargetBitrateBps),
1351 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001352
perkja49cbd32016-09-16 07:53:41 -07001353 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001354 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001355
mflodmancc3d4422017-08-03 08:27:51 -07001356 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001357 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001358 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001359 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1360 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001361}
1362
mflodmancc3d4422017-08-03 08:27:51 -07001363TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001365 DataRate::BitsPerSec(kTargetBitrateBps),
1366 DataRate::BitsPerSec(kTargetBitrateBps),
1367 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001368
1369 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001370 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001371 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001372 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1373 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001374 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1375 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001376 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001377 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001378
mflodmancc3d4422017-08-03 08:27:51 -07001379 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001380}
1381
Noah Richards51db4212019-06-12 06:59:12 -07001382TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001384 DataRate::BitsPerSec(kTargetBitrateBps),
1385 DataRate::BitsPerSec(kTargetBitrateBps),
1386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001387
1388 rtc::Event frame_destroyed_event;
1389 video_source_.IncomingCapturedFrame(
1390 CreateFakeNativeFrame(1, &frame_destroyed_event));
1391 ExpectDroppedFrame();
1392 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1393 video_stream_encoder_->Stop();
1394}
1395
1396TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1397 // Use the cropping factory.
1398 video_encoder_config_.video_stream_factory =
1399 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1400 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1401 kMaxPayloadLength);
1402 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1403
1404 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001405 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001406 DataRate::BitsPerSec(kTargetBitrateBps),
1407 DataRate::BitsPerSec(kTargetBitrateBps),
1408 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001409 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1410 WaitForEncodedFrame(1);
1411 // The encoder will have been configured once.
1412 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1413 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1414 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1415
1416 // Now send in a fake frame that needs to be cropped as the width/height
1417 // aren't divisible by 4 (see CreateEncoderStreams above).
1418 rtc::Event frame_destroyed_event;
1419 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1420 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1421 ExpectDroppedFrame();
1422 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1423 video_stream_encoder_->Stop();
1424}
1425
Ying Wang9b881ab2020-02-07 14:29:32 +01001426TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001428 DataRate::BitsPerSec(kTargetBitrateBps),
1429 DataRate::BitsPerSec(kTargetBitrateBps),
1430 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001431 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1432 WaitForEncodedFrame(1);
1433
Henrik Boström381d1092020-05-12 18:49:07 +02001434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001435 DataRate::BitsPerSec(kTargetBitrateBps),
1436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001438 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1439 // frames. Adding two frames means that the first frame will be dropped and
1440 // the second frame will be sent to the encoder.
1441 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1442 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1443 WaitForEncodedFrame(3);
1444 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1445 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1446 WaitForEncodedFrame(5);
1447 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1448 video_stream_encoder_->Stop();
1449}
1450
mflodmancc3d4422017-08-03 08:27:51 -07001451TEST_F(VideoStreamEncoderTest,
1452 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001454 DataRate::BitsPerSec(kTargetBitrateBps),
1455 DataRate::BitsPerSec(kTargetBitrateBps),
1456 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001457 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001458
1459 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001460 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001462 // The encoder will have been configured once when the first frame is
1463 // received.
1464 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001465
1466 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001467 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001468 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001469 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001470 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001471
1472 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001473 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001474 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001475 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001476 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001477
mflodmancc3d4422017-08-03 08:27:51 -07001478 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001479}
1480
mflodmancc3d4422017-08-03 08:27:51 -07001481TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001483 DataRate::BitsPerSec(kTargetBitrateBps),
1484 DataRate::BitsPerSec(kTargetBitrateBps),
1485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001486
1487 // Capture a frame and wait for it to synchronize with the encoder thread.
1488 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001489 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001490 // The encoder will have been configured once.
1491 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001492 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1493 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1494
1495 codec_width_ *= 2;
1496 codec_height_ *= 2;
1497 // Capture a frame with a higher resolution and wait for it to synchronize
1498 // with the encoder thread.
1499 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001500 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001501 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1502 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001503 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001504
mflodmancc3d4422017-08-03 08:27:51 -07001505 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001506}
1507
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001508TEST_F(VideoStreamEncoderTest,
1509 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001511 DataRate::BitsPerSec(kTargetBitrateBps),
1512 DataRate::BitsPerSec(kTargetBitrateBps),
1513 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001514
1515 // Capture a frame and wait for it to synchronize with the encoder thread.
1516 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1517 WaitForEncodedFrame(1);
1518
1519 VideoEncoderConfig video_encoder_config;
1520 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1521 // Changing the max payload data length recreates encoder.
1522 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1523 kMaxPayloadLength / 2);
1524
1525 // Capture a frame and wait for it to synchronize with the encoder thread.
1526 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1527 WaitForEncodedFrame(2);
1528 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1529
1530 video_stream_encoder_->Stop();
1531}
1532
Sergey Silkin5ee69672019-07-02 14:18:34 +02001533TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001538
1539 VideoEncoderConfig video_encoder_config;
1540 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1541 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1542 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1543 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1544 kMaxPayloadLength);
1545
1546 // Capture a frame and wait for it to synchronize with the encoder thread.
1547 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1548 WaitForEncodedFrame(1);
1549 // The encoder will have been configured once when the first frame is
1550 // received.
1551 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1552 EXPECT_EQ(kTargetBitrateBps,
1553 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1554 EXPECT_EQ(kStartBitrateBps,
1555 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1556
Sergey Silkin6456e352019-07-08 17:56:40 +02001557 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1558 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001559 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1560 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1561 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1562 kMaxPayloadLength);
1563
1564 // Capture a frame and wait for it to synchronize with the encoder thread.
1565 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1566 WaitForEncodedFrame(2);
1567 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1568 // Bitrate limits have changed - rate allocator should be reconfigured,
1569 // encoder should not be reconfigured.
1570 EXPECT_EQ(kTargetBitrateBps * 2,
1571 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1572 EXPECT_EQ(kStartBitrateBps * 2,
1573 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1574 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1575
1576 video_stream_encoder_->Stop();
1577}
1578
Sergey Silkin6456e352019-07-08 17:56:40 +02001579TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001580 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001582 DataRate::BitsPerSec(kTargetBitrateBps),
1583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001585
Sergey Silkincd02eba2020-01-20 14:48:40 +01001586 const uint32_t kMinEncBitrateKbps = 100;
1587 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001588 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001589 /*frame_size_pixels=*/codec_width_ * codec_height_,
1590 /*min_start_bitrate_bps=*/0,
1591 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1592 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001593 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1594
Sergey Silkincd02eba2020-01-20 14:48:40 +01001595 VideoEncoderConfig video_encoder_config;
1596 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1597 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1598 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1599 (kMinEncBitrateKbps + 1) * 1000;
1600 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1601 kMaxPayloadLength);
1602
1603 // When both encoder and app provide bitrate limits, the intersection of
1604 // provided sets should be used.
1605 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1606 WaitForEncodedFrame(1);
1607 EXPECT_EQ(kMaxEncBitrateKbps,
1608 bitrate_allocator_factory_.codec_config().maxBitrate);
1609 EXPECT_EQ(kMinEncBitrateKbps + 1,
1610 bitrate_allocator_factory_.codec_config().minBitrate);
1611
1612 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1613 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1614 (kMinEncBitrateKbps - 1) * 1000;
1615 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1616 kMaxPayloadLength);
1617 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001618 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001619 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001620 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001621 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001622 bitrate_allocator_factory_.codec_config().minBitrate);
1623
Sergey Silkincd02eba2020-01-20 14:48:40 +01001624 video_stream_encoder_->Stop();
1625}
1626
1627TEST_F(VideoStreamEncoderTest,
1628 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001630 DataRate::BitsPerSec(kTargetBitrateBps),
1631 DataRate::BitsPerSec(kTargetBitrateBps),
1632 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001633
1634 const uint32_t kMinAppBitrateKbps = 100;
1635 const uint32_t kMaxAppBitrateKbps = 200;
1636 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1637 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1638 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1639 /*frame_size_pixels=*/codec_width_ * codec_height_,
1640 /*min_start_bitrate_bps=*/0,
1641 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1642 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1643 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1644
1645 VideoEncoderConfig video_encoder_config;
1646 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1647 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1648 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1649 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001650 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1651 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001652
Sergey Silkincd02eba2020-01-20 14:48:40 +01001653 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1654 WaitForEncodedFrame(1);
1655 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001656 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001657 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001658 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001659
1660 video_stream_encoder_->Stop();
1661}
1662
1663TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001664 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001666 DataRate::BitsPerSec(kTargetBitrateBps),
1667 DataRate::BitsPerSec(kTargetBitrateBps),
1668 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001669
1670 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001671 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001672 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001673 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001674 fake_encoder_.SetResolutionBitrateLimits(
1675 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1676
1677 VideoEncoderConfig video_encoder_config;
1678 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1679 video_encoder_config.max_bitrate_bps = 0;
1680 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1681 kMaxPayloadLength);
1682
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001683 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001684 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1685 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001686 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1687 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001688 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1689 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1690
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001691 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001692 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1693 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001694 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1695 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001696 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1697 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1698
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001699 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001700 // encoder for 360p should be used.
1701 video_source_.IncomingCapturedFrame(
1702 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1703 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001704 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1705 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001706 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1707 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1708
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001709 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001710 // ignored.
1711 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1712 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001713 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1714 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001715 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1716 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001717 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1718 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001719 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1720 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1721
1722 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1723 // for 270p should be used.
1724 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1725 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001726 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1727 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001728 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1729 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1730
1731 video_stream_encoder_->Stop();
1732}
1733
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001734TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001736 DataRate::BitsPerSec(kTargetBitrateBps),
1737 DataRate::BitsPerSec(kTargetBitrateBps),
1738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001739
1740 VideoEncoderConfig video_encoder_config;
1741 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1742 video_encoder_config.max_bitrate_bps = 0;
1743 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1744 kMaxPayloadLength);
1745
1746 // Encode 720p frame to get the default encoder target bitrate.
1747 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1748 WaitForEncodedFrame(1);
1749 const uint32_t kDefaultTargetBitrateFor720pKbps =
1750 bitrate_allocator_factory_.codec_config()
1751 .simulcastStream[0]
1752 .targetBitrate;
1753
1754 // Set the max recommended encoder bitrate to something lower than the default
1755 // target bitrate.
1756 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1757 1280 * 720, 10 * 1000, 10 * 1000,
1758 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1759 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1760
1761 // Change resolution to trigger encoder reinitialization.
1762 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1763 WaitForEncodedFrame(2);
1764 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1765 WaitForEncodedFrame(3);
1766
1767 // Ensure the target bitrate is capped by the max bitrate.
1768 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1769 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1770 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1771 .simulcastStream[0]
1772 .targetBitrate *
1773 1000,
1774 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1775
1776 video_stream_encoder_->Stop();
1777}
1778
mflodmancc3d4422017-08-03 08:27:51 -07001779TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001780 EXPECT_TRUE(video_source_.has_sinks());
1781 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001782 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001783 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001784 EXPECT_FALSE(video_source_.has_sinks());
1785 EXPECT_TRUE(new_video_source.has_sinks());
1786
mflodmancc3d4422017-08-03 08:27:51 -07001787 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001788}
1789
mflodmancc3d4422017-08-03 08:27:51 -07001790TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001791 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001792 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001793 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001794 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001795}
1796
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001797TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1798 constexpr int kRequestedResolutionAlignment = 7;
1799 video_source_.set_adaptation_enabled(true);
1800 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001802 DataRate::BitsPerSec(kTargetBitrateBps),
1803 DataRate::BitsPerSec(kTargetBitrateBps),
1804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001805
1806 // On the 1st frame, we should have initialized the encoder and
1807 // asked for its resolution requirements.
1808 video_source_.IncomingCapturedFrame(
1809 CreateFrame(1, codec_width_, codec_height_));
1810 WaitForEncodedFrame(1);
1811 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1812 kRequestedResolutionAlignment);
1813
1814 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1815 // (It's up the to the encoder to potentially drop the previous frame,
1816 // to avoid coding back-to-back keyframes.)
1817 video_source_.IncomingCapturedFrame(
1818 CreateFrame(2, codec_width_, codec_height_));
1819 WaitForEncodedFrame(2);
1820 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1821
1822 video_stream_encoder_->Stop();
1823}
1824
Jonathan Yubc771b72017-12-08 17:04:29 -08001825TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1826 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001827 const int kWidth = 1280;
1828 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001829
1830 // We rely on the automatic resolution adaptation, but we handle framerate
1831 // adaptation manually by mocking the stats proxy.
1832 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001833
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001834 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001836 DataRate::BitsPerSec(kTargetBitrateBps),
1837 DataRate::BitsPerSec(kTargetBitrateBps),
1838 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001839 video_stream_encoder_->SetSource(&video_source_,
1840 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001841 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1845
Jonathan Yubc771b72017-12-08 17:04:29 -08001846 // Adapt down as far as possible.
1847 rtc::VideoSinkWants last_wants;
1848 int64_t t = 1;
1849 int loop_count = 0;
1850 do {
1851 ++loop_count;
1852 last_wants = video_source_.sink_wants();
1853
1854 // Simulate the framerate we've been asked to adapt to.
1855 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1856 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1857 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1858 mock_stats.input_frame_rate = fps;
1859 stats_proxy_->SetMockStats(mock_stats);
1860
1861 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1862 sink_.WaitForEncodedFrame(t);
1863 t += frame_interval_ms;
1864
mflodmancc3d4422017-08-03 08:27:51 -07001865 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001866 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001867 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001868 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1869 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001870 } while (video_source_.sink_wants().max_pixel_count <
1871 last_wants.max_pixel_count ||
1872 video_source_.sink_wants().max_framerate_fps <
1873 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001874
Jonathan Yubc771b72017-12-08 17:04:29 -08001875 // Verify that we've adapted all the way down.
1876 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001877 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001878 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1879 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001880 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001881 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1882 *video_source_.last_sent_height());
1883 EXPECT_EQ(kMinBalancedFramerateFps,
1884 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001885
Jonathan Yubc771b72017-12-08 17:04:29 -08001886 // Adapt back up the same number of times we adapted down.
1887 for (int i = 0; i < loop_count - 1; ++i) {
1888 last_wants = video_source_.sink_wants();
1889
1890 // Simulate the framerate we've been asked to adapt to.
1891 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1892 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1893 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1894 mock_stats.input_frame_rate = fps;
1895 stats_proxy_->SetMockStats(mock_stats);
1896
1897 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1898 sink_.WaitForEncodedFrame(t);
1899 t += frame_interval_ms;
1900
Henrik Boström91aa7322020-04-28 12:24:33 +02001901 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001902 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001903 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001904 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1905 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001906 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1907 last_wants.max_pixel_count ||
1908 video_source_.sink_wants().max_framerate_fps >
1909 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001910 }
1911
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001912 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001913 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001914 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001915 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1916 EXPECT_EQ((loop_count - 1) * 2,
1917 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001918
mflodmancc3d4422017-08-03 08:27:51 -07001919 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001920}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001921
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001922TEST_F(VideoStreamEncoderTest,
1923 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1924 video_stream_encoder_->OnBitrateUpdated(
1925 DataRate::BitsPerSec(kTargetBitrateBps),
1926 DataRate::BitsPerSec(kTargetBitrateBps),
1927 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001928 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001929
1930 const int kFrameWidth = 1280;
1931 const int kFrameHeight = 720;
1932
1933 int64_t ntp_time = kFrameIntervalMs;
1934
1935 // Force an input frame rate to be available, or the adaptation call won't
1936 // know what framerate to adapt form.
1937 const int kInputFps = 30;
1938 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1939 stats.input_frame_rate = kInputFps;
1940 stats_proxy_->SetMockStats(stats);
1941
1942 video_source_.set_adaptation_enabled(true);
1943 video_stream_encoder_->SetSource(
1944 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001945 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001946 video_source_.IncomingCapturedFrame(
1947 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1948 sink_.WaitForEncodedFrame(ntp_time);
1949 ntp_time += kFrameIntervalMs;
1950
1951 // Trigger CPU overuse.
1952 video_stream_encoder_->TriggerCpuOveruse();
1953 video_source_.IncomingCapturedFrame(
1954 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1955 sink_.WaitForEncodedFrame(ntp_time);
1956 ntp_time += kFrameIntervalMs;
1957
1958 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1959 EXPECT_EQ(std::numeric_limits<int>::max(),
1960 video_source_.sink_wants().max_pixel_count);
1961 // Some framerate constraint should be set.
1962 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1963 EXPECT_LT(restricted_fps, kInputFps);
1964 video_source_.IncomingCapturedFrame(
1965 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1966 sink_.WaitForEncodedFrame(ntp_time);
1967 ntp_time += 100;
1968
Henrik Boström2671dac2020-05-19 16:29:09 +02001969 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001970 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
1971 // Give the encoder queue time to process the change in degradation preference
1972 // by waiting for an encoded frame.
1973 video_source_.IncomingCapturedFrame(
1974 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1975 sink_.WaitForEncodedFrame(ntp_time);
1976 ntp_time += kFrameIntervalMs;
1977
1978 video_stream_encoder_->TriggerQualityLow();
1979 video_source_.IncomingCapturedFrame(
1980 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1981 sink_.WaitForEncodedFrame(ntp_time);
1982 ntp_time += kFrameIntervalMs;
1983
1984 // Some resolution constraint should be set.
1985 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1986 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1987 kFrameWidth * kFrameHeight);
1988 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
1989
1990 int pixel_count = video_source_.sink_wants().max_pixel_count;
1991 // Triggering a CPU underuse should not change the sink wants since it has
1992 // not been overused for resolution since we changed degradation preference.
1993 video_stream_encoder_->TriggerCpuUnderuse();
1994 video_source_.IncomingCapturedFrame(
1995 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1996 sink_.WaitForEncodedFrame(ntp_time);
1997 ntp_time += kFrameIntervalMs;
1998 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
1999 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2000
Evan Shrubsole64469032020-06-11 10:45:29 +02002001 // Change the degradation preference back. CPU underuse should not adapt since
2002 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002003 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002004 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2005 video_source_.IncomingCapturedFrame(
2006 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2007 sink_.WaitForEncodedFrame(ntp_time);
2008 ntp_time += 100;
2009 // Resolution adaptations is gone after changing degradation preference.
2010 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2011 EXPECT_EQ(std::numeric_limits<int>::max(),
2012 video_source_.sink_wants().max_pixel_count);
2013 // The fps adaptation from above is now back.
2014 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2015
2016 // Trigger CPU underuse.
2017 video_stream_encoder_->TriggerCpuUnderuse();
2018 video_source_.IncomingCapturedFrame(
2019 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2020 sink_.WaitForEncodedFrame(ntp_time);
2021 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002022 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2023
2024 // Trigger QP underuse, fps should return to normal.
2025 video_stream_encoder_->TriggerQualityHigh();
2026 video_source_.IncomingCapturedFrame(
2027 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2028 sink_.WaitForEncodedFrame(ntp_time);
2029 ntp_time += kFrameIntervalMs;
2030 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002031
2032 video_stream_encoder_->Stop();
2033}
2034
mflodmancc3d4422017-08-03 08:27:51 -07002035TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002037 DataRate::BitsPerSec(kTargetBitrateBps),
2038 DataRate::BitsPerSec(kTargetBitrateBps),
2039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002040 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002041
sprangc5d62e22017-04-02 23:53:04 -07002042 const int kFrameWidth = 1280;
2043 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002044
Åsa Persson8c1bf952018-09-13 10:42:19 +02002045 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002046
kthelgason5e13d412016-12-01 03:59:51 -08002047 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002048 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002049 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002050 frame_timestamp += kFrameIntervalMs;
2051
perkj803d97f2016-11-01 11:45:46 -07002052 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002053 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002054 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002055 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002056 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002057 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002058
asapersson0944a802017-04-07 00:57:58 -07002059 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002060 // wanted resolution.
2061 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2062 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2063 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002064 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002065
2066 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002067 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002068 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002069 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002070 // Give the encoder queue time to process the change in degradation preference
2071 // by waiting for an encoded frame.
2072 new_video_source.IncomingCapturedFrame(
2073 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2074 sink_.WaitForEncodedFrame(frame_timestamp);
2075 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002076 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002077 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002078
sprangc5d62e22017-04-02 23:53:04 -07002079 // Force an input frame rate to be available, or the adaptation call won't
2080 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002081 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002082 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002083 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002084 stats_proxy_->SetMockStats(stats);
2085
mflodmancc3d4422017-08-03 08:27:51 -07002086 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002087 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002088 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002089 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002090 frame_timestamp += kFrameIntervalMs;
2091
2092 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002093 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002094 EXPECT_EQ(std::numeric_limits<int>::max(),
2095 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002096 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002097
asapersson02465b82017-04-10 01:12:52 -07002098 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002099 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2100 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002101 // Give the encoder queue time to process the change in degradation preference
2102 // by waiting for an encoded frame.
2103 new_video_source.IncomingCapturedFrame(
2104 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2105 sink_.WaitForEncodedFrame(frame_timestamp);
2106 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002107 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002108
mflodmancc3d4422017-08-03 08:27:51 -07002109 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002110 new_video_source.IncomingCapturedFrame(
2111 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002112 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002113 frame_timestamp += kFrameIntervalMs;
2114
2115 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002116 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002117
2118 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002119 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002120 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002121 // Give the encoder queue time to process the change in degradation preference
2122 // by waiting for an encoded frame.
2123 new_video_source.IncomingCapturedFrame(
2124 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2125 sink_.WaitForEncodedFrame(frame_timestamp);
2126 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002127 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2128 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002129 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002130 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002131
2132 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002133 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002134 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002135 // Give the encoder queue time to process the change in degradation preference
2136 // by waiting for an encoded frame.
2137 new_video_source.IncomingCapturedFrame(
2138 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2139 sink_.WaitForEncodedFrame(frame_timestamp);
2140 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002141 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2142 EXPECT_EQ(std::numeric_limits<int>::max(),
2143 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002144 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002145
mflodmancc3d4422017-08-03 08:27:51 -07002146 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002147}
2148
mflodmancc3d4422017-08-03 08:27:51 -07002149TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002151 DataRate::BitsPerSec(kTargetBitrateBps),
2152 DataRate::BitsPerSec(kTargetBitrateBps),
2153 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002154
asaperssonfab67072017-04-04 05:51:49 -07002155 const int kWidth = 1280;
2156 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002157 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002158 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002159 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2160 EXPECT_FALSE(stats.bw_limited_resolution);
2161 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2162
2163 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002164 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002165 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002166 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002167
2168 stats = stats_proxy_->GetStats();
2169 EXPECT_TRUE(stats.bw_limited_resolution);
2170 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2171
2172 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002173 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002174 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002175 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002176
2177 stats = stats_proxy_->GetStats();
2178 EXPECT_FALSE(stats.bw_limited_resolution);
2179 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2180 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2181
mflodmancc3d4422017-08-03 08:27:51 -07002182 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002183}
2184
mflodmancc3d4422017-08-03 08:27:51 -07002185TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002186 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002187 DataRate::BitsPerSec(kTargetBitrateBps),
2188 DataRate::BitsPerSec(kTargetBitrateBps),
2189 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002190
2191 const int kWidth = 1280;
2192 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002193 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002194 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002195 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2196 EXPECT_FALSE(stats.cpu_limited_resolution);
2197 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2198
2199 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002200 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002201 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002202 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002203
2204 stats = stats_proxy_->GetStats();
2205 EXPECT_TRUE(stats.cpu_limited_resolution);
2206 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2207
2208 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002209 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002210 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002211 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002212
2213 stats = stats_proxy_->GetStats();
2214 EXPECT_FALSE(stats.cpu_limited_resolution);
2215 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002216 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002217
mflodmancc3d4422017-08-03 08:27:51 -07002218 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002219}
2220
mflodmancc3d4422017-08-03 08:27:51 -07002221TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002222 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002223 DataRate::BitsPerSec(kTargetBitrateBps),
2224 DataRate::BitsPerSec(kTargetBitrateBps),
2225 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002226
asaperssonfab67072017-04-04 05:51:49 -07002227 const int kWidth = 1280;
2228 const int kHeight = 720;
2229 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002230 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002231 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002232 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002233 EXPECT_FALSE(stats.cpu_limited_resolution);
2234 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2235
asaperssonfab67072017-04-04 05:51:49 -07002236 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002237 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002238 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002239 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002240 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002241 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002242 EXPECT_TRUE(stats.cpu_limited_resolution);
2243 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2244
2245 // Set new source with adaptation still enabled.
2246 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002247 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002248 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002249
asaperssonfab67072017-04-04 05:51:49 -07002250 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002251 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002252 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002253 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002254 EXPECT_TRUE(stats.cpu_limited_resolution);
2255 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2256
2257 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002258 video_stream_encoder_->SetSource(&new_video_source,
2259 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002260
asaperssonfab67072017-04-04 05:51:49 -07002261 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002262 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002263 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002264 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002265 EXPECT_FALSE(stats.cpu_limited_resolution);
2266 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2267
2268 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002269 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002270 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002271
asaperssonfab67072017-04-04 05:51:49 -07002272 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002273 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002274 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002275 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002276 EXPECT_TRUE(stats.cpu_limited_resolution);
2277 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2278
asaperssonfab67072017-04-04 05:51:49 -07002279 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002280 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002281 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002282 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002283 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002284 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002285 EXPECT_FALSE(stats.cpu_limited_resolution);
2286 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002287 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002288
mflodmancc3d4422017-08-03 08:27:51 -07002289 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002290}
2291
mflodmancc3d4422017-08-03 08:27:51 -07002292TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002294 DataRate::BitsPerSec(kTargetBitrateBps),
2295 DataRate::BitsPerSec(kTargetBitrateBps),
2296 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002297
asaperssonfab67072017-04-04 05:51:49 -07002298 const int kWidth = 1280;
2299 const int kHeight = 720;
2300 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002301 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002302 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002303 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002304 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002305 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002306
2307 // Set new source with adaptation still enabled.
2308 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002309 video_stream_encoder_->SetSource(&new_video_source,
2310 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002311
asaperssonfab67072017-04-04 05:51:49 -07002312 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002313 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002314 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002315 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002316 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002317 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002318
asaperssonfab67072017-04-04 05:51:49 -07002319 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002320 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002321 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002322 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002323 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002324 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002325 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002326 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002327
asaperssonfab67072017-04-04 05:51:49 -07002328 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002329 video_stream_encoder_->SetSource(&new_video_source,
2330 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002331
asaperssonfab67072017-04-04 05:51:49 -07002332 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002333 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002334 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002335 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002336 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002337 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002338
asapersson02465b82017-04-10 01:12:52 -07002339 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002340 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002341 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002342
asaperssonfab67072017-04-04 05:51:49 -07002343 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002344 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002345 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002346 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002347 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002348 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2349 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002350
mflodmancc3d4422017-08-03 08:27:51 -07002351 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002352}
2353
mflodmancc3d4422017-08-03 08:27:51 -07002354TEST_F(VideoStreamEncoderTest,
2355 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002357 DataRate::BitsPerSec(kTargetBitrateBps),
2358 DataRate::BitsPerSec(kTargetBitrateBps),
2359 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002360
2361 const int kWidth = 1280;
2362 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002363 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002364 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002365 video_source_.IncomingCapturedFrame(
2366 CreateFrame(timestamp_ms, kWidth, kHeight));
2367 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002368 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2369 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2370 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2371
2372 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002373 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002374 timestamp_ms += kFrameIntervalMs;
2375 video_source_.IncomingCapturedFrame(
2376 CreateFrame(timestamp_ms, kWidth, kHeight));
2377 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002378 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2380 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2381
2382 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002383 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002384 timestamp_ms += kFrameIntervalMs;
2385 video_source_.IncomingCapturedFrame(
2386 CreateFrame(timestamp_ms, kWidth, kHeight));
2387 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002388 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2391
Niels Möller4db138e2018-04-19 09:04:13 +02002392 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002393 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002394
2395 VideoEncoderConfig video_encoder_config;
2396 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2397 // Make format different, to force recreation of encoder.
2398 video_encoder_config.video_format.parameters["foo"] = "foo";
2399 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002400 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002401 timestamp_ms += kFrameIntervalMs;
2402 video_source_.IncomingCapturedFrame(
2403 CreateFrame(timestamp_ms, kWidth, kHeight));
2404 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002405 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2407 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2408
mflodmancc3d4422017-08-03 08:27:51 -07002409 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002410}
2411
mflodmancc3d4422017-08-03 08:27:51 -07002412TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002413 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002415 DataRate::BitsPerSec(kTargetBitrateBps),
2416 DataRate::BitsPerSec(kTargetBitrateBps),
2417 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2418
2419 const int kWidth = 1280;
2420 const int kHeight = 720;
2421 int sequence = 1;
2422
2423 // Enable BALANCED preference, no initial limitation.
2424 test::FrameForwarder source;
2425 video_stream_encoder_->SetSource(&source,
2426 webrtc::DegradationPreference::BALANCED);
2427 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2428 WaitForEncodedFrame(sequence++);
2429 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2430 EXPECT_FALSE(stats.cpu_limited_resolution);
2431 EXPECT_FALSE(stats.cpu_limited_framerate);
2432 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2433
2434 // Trigger CPU overuse, should now adapt down.
2435 video_stream_encoder_->TriggerCpuOveruse();
2436 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2437 WaitForEncodedFrame(sequence++);
2438 stats = stats_proxy_->GetStats();
2439 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2440
2441 // Set new degradation preference should clear restrictions since we changed
2442 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002443 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002444 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2445 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2446 WaitForEncodedFrame(sequence++);
2447 stats = stats_proxy_->GetStats();
2448 EXPECT_FALSE(stats.cpu_limited_resolution);
2449 EXPECT_FALSE(stats.cpu_limited_framerate);
2450 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2451
2452 // Force an input frame rate to be available, or the adaptation call won't
2453 // know what framerate to adapt from.
2454 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2455 mock_stats.input_frame_rate = 30;
2456 stats_proxy_->SetMockStats(mock_stats);
2457 video_stream_encoder_->TriggerCpuOveruse();
2458 stats_proxy_->ResetMockStats();
2459 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2460 WaitForEncodedFrame(sequence++);
2461
2462 // We have now adapted once.
2463 stats = stats_proxy_->GetStats();
2464 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2465
2466 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002467 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2468 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002469 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2470 WaitForEncodedFrame(sequence++);
2471 stats = stats_proxy_->GetStats();
2472 EXPECT_FALSE(stats.cpu_limited_resolution);
2473 EXPECT_FALSE(stats.cpu_limited_framerate);
2474 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2475
2476 video_stream_encoder_->Stop();
2477}
2478
2479TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002480 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002482 DataRate::BitsPerSec(kTargetBitrateBps),
2483 DataRate::BitsPerSec(kTargetBitrateBps),
2484 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002485
asapersson0944a802017-04-07 00:57:58 -07002486 const int kWidth = 1280;
2487 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002488 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002489
asaperssonfab67072017-04-04 05:51:49 -07002490 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002491 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002492 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002493 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002494 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002495 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2496
asapersson02465b82017-04-10 01:12:52 -07002497 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002498 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002499 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002500 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002501 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002502 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002503 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002504 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2505
2506 // Set new source with adaptation still enabled.
2507 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002508 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002509 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002510
2511 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002512 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002513 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002514 stats = stats_proxy_->GetStats();
2515 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002516 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002517 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2518
sprangc5d62e22017-04-02 23:53:04 -07002519 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002520 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002521 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002522 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002523 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002524 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002525 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002526 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002527 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002528 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002529 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2530
sprangc5d62e22017-04-02 23:53:04 -07002531 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002532 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002533 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2534 mock_stats.input_frame_rate = 30;
2535 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002536 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002537 stats_proxy_->ResetMockStats();
2538
2539 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002540 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002541 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002542
2543 // Framerate now adapted.
2544 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002545 EXPECT_FALSE(stats.cpu_limited_resolution);
2546 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002547 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2548
2549 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002550 video_stream_encoder_->SetSource(&new_video_source,
2551 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002552 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002553 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002554 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002555
2556 stats = stats_proxy_->GetStats();
2557 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002558 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002559 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2560
2561 // Try to trigger overuse. Should not succeed.
2562 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002563 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002564 stats_proxy_->ResetMockStats();
2565
2566 stats = stats_proxy_->GetStats();
2567 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002568 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002569 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2570
2571 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002572 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002573 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002574 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002575 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002576 stats = stats_proxy_->GetStats();
2577 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002578 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002579 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002580
2581 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002582 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002583 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002584 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002585 stats = stats_proxy_->GetStats();
2586 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002587 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002588 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2589
2590 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002591 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002592 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002593 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002594 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002595 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002596 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002597 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002598 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002599 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002600 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2601
2602 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002603 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002604 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002605 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002606 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002607 stats = stats_proxy_->GetStats();
2608 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002609 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002610 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002611 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002612
mflodmancc3d4422017-08-03 08:27:51 -07002613 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002614}
2615
mflodmancc3d4422017-08-03 08:27:51 -07002616TEST_F(VideoStreamEncoderTest,
2617 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002618 const int kWidth = 1280;
2619 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002621 DataRate::BitsPerSec(kTargetBitrateBps),
2622 DataRate::BitsPerSec(kTargetBitrateBps),
2623 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002624
asaperssonfab67072017-04-04 05:51:49 -07002625 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002626 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002627
asaperssonfab67072017-04-04 05:51:49 -07002628 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002629 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002630
asaperssonfab67072017-04-04 05:51:49 -07002631 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002633
asaperssonfab67072017-04-04 05:51:49 -07002634 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002635 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002636
kthelgason876222f2016-11-29 01:44:11 -08002637 // Expect a scale down.
2638 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002639 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002640
asapersson02465b82017-04-10 01:12:52 -07002641 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002642 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002643 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002644 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002645
asaperssonfab67072017-04-04 05:51:49 -07002646 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002647 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002648 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002649 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002650
asaperssonfab67072017-04-04 05:51:49 -07002651 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002652 EXPECT_EQ(std::numeric_limits<int>::max(),
2653 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002654
asaperssonfab67072017-04-04 05:51:49 -07002655 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002656 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002657 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002658 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002659
asapersson02465b82017-04-10 01:12:52 -07002660 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002661 EXPECT_EQ(std::numeric_limits<int>::max(),
2662 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002663
mflodmancc3d4422017-08-03 08:27:51 -07002664 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002665}
2666
mflodmancc3d4422017-08-03 08:27:51 -07002667TEST_F(VideoStreamEncoderTest,
2668 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002669 const int kWidth = 1280;
2670 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002672 DataRate::BitsPerSec(kTargetBitrateBps),
2673 DataRate::BitsPerSec(kTargetBitrateBps),
2674 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002675
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002676 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002677 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002678 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002679 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002680
2681 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002682 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2685 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2686
2687 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002689 EXPECT_THAT(source.sink_wants(),
2690 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002691 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2692 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2693 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2694
2695 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002696 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002697 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2698 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2699 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2700
mflodmancc3d4422017-08-03 08:27:51 -07002701 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002702}
2703
mflodmancc3d4422017-08-03 08:27:51 -07002704TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002705 const int kWidth = 1280;
2706 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002708 DataRate::BitsPerSec(kTargetBitrateBps),
2709 DataRate::BitsPerSec(kTargetBitrateBps),
2710 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002711
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002712 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002713 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002714 video_stream_encoder_->SetSource(&source,
2715 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002716 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2717 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002719
2720 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002721 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002722 EXPECT_THAT(source.sink_wants(),
2723 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002724 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2725 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2726 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2727
2728 // Trigger adapt down for same input resolution, expect no change.
2729 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2730 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002732 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2733 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2734 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2735
2736 // Trigger adapt down for larger input resolution, expect no change.
2737 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2738 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002739 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002740 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2741 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2742 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2743
mflodmancc3d4422017-08-03 08:27:51 -07002744 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002745}
2746
mflodmancc3d4422017-08-03 08:27:51 -07002747TEST_F(VideoStreamEncoderTest,
2748 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002749 const int kWidth = 1280;
2750 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002752 DataRate::BitsPerSec(kTargetBitrateBps),
2753 DataRate::BitsPerSec(kTargetBitrateBps),
2754 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002755
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002756 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002757 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002758 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002759 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002760
2761 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002762 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002763 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2765 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2766
2767 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002768 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002769 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002770 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2771 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2772
mflodmancc3d4422017-08-03 08:27:51 -07002773 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002774}
2775
mflodmancc3d4422017-08-03 08:27:51 -07002776TEST_F(VideoStreamEncoderTest,
2777 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002778 const int kWidth = 1280;
2779 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002781 DataRate::BitsPerSec(kTargetBitrateBps),
2782 DataRate::BitsPerSec(kTargetBitrateBps),
2783 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002784
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002785 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002786 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002787 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002788 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002789
2790 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002791 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002792 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002793 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002794 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2795
2796 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002797 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2801
mflodmancc3d4422017-08-03 08:27:51 -07002802 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002803}
2804
mflodmancc3d4422017-08-03 08:27:51 -07002805TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002806 const int kWidth = 1280;
2807 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002809 DataRate::BitsPerSec(kTargetBitrateBps),
2810 DataRate::BitsPerSec(kTargetBitrateBps),
2811 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002812
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002813 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002814 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002815 video_stream_encoder_->SetSource(&source,
2816 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002817
2818 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2819 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002820 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2822 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2823 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2824
2825 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002826 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002827 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2829 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2831
mflodmancc3d4422017-08-03 08:27:51 -07002832 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002833}
2834
mflodmancc3d4422017-08-03 08:27:51 -07002835TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002836 const int kWidth = 1280;
2837 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002839 DataRate::BitsPerSec(kTargetBitrateBps),
2840 DataRate::BitsPerSec(kTargetBitrateBps),
2841 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002843 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002844 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002845 video_stream_encoder_->SetSource(&source,
2846 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002847
2848 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2849 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002850 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2852 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2853 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2854
2855 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002856 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002857 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002858 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2859 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2860 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2861
mflodmancc3d4422017-08-03 08:27:51 -07002862 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002863}
2864
mflodmancc3d4422017-08-03 08:27:51 -07002865TEST_F(VideoStreamEncoderTest,
2866 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002867 const int kWidth = 1280;
2868 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002870 DataRate::BitsPerSec(kTargetBitrateBps),
2871 DataRate::BitsPerSec(kTargetBitrateBps),
2872 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002873
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002874 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002875 AdaptingFrameForwarder source;
2876 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002878 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002879
2880 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002881 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002882 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2884 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2885
2886 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002887 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002888 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002889 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002890 EXPECT_THAT(source.sink_wants(),
2891 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002892 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2894
2895 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002896 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002897 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2899 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2900 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2901
mflodmancc3d4422017-08-03 08:27:51 -07002902 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002903}
2904
mflodmancc3d4422017-08-03 08:27:51 -07002905TEST_F(VideoStreamEncoderTest,
2906 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002907 const int kWidth = 1280;
2908 const int kHeight = 720;
2909 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002911 DataRate::BitsPerSec(kTargetBitrateBps),
2912 DataRate::BitsPerSec(kTargetBitrateBps),
2913 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002914
2915 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2916 stats.input_frame_rate = kInputFps;
2917 stats_proxy_->SetMockStats(stats);
2918
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002919 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002920 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2921 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002922 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002923
2924 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002925 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002926 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2927 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002928 EXPECT_THAT(video_source_.sink_wants(),
2929 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002930
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002931 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002932 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002933 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002934 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002935 // Give the encoder queue time to process the change in degradation preference
2936 // by waiting for an encoded frame.
2937 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2938 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002939 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002940
2941 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002942 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002943 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2944 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002945 EXPECT_THAT(new_video_source.sink_wants(),
2946 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002947
2948 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002949 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002950 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002951
mflodmancc3d4422017-08-03 08:27:51 -07002952 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002953}
2954
mflodmancc3d4422017-08-03 08:27:51 -07002955TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002956 const int kWidth = 1280;
2957 const int kHeight = 720;
2958 const size_t kNumFrames = 10;
2959
Henrik Boström381d1092020-05-12 18:49:07 +02002960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002961 DataRate::BitsPerSec(kTargetBitrateBps),
2962 DataRate::BitsPerSec(kTargetBitrateBps),
2963 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002964
asaperssond0de2952017-04-21 01:47:31 -07002965 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002966 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002967 video_source_.set_adaptation_enabled(true);
2968
2969 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2970 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2971
2972 int downscales = 0;
2973 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002974 video_source_.IncomingCapturedFrame(
2975 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2976 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002977
asaperssonfab67072017-04-04 05:51:49 -07002978 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002979 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002980 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002981 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002982
2983 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2984 ++downscales;
2985
2986 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2987 EXPECT_EQ(downscales,
2988 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2989 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002990 }
mflodmancc3d4422017-08-03 08:27:51 -07002991 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002992}
2993
mflodmancc3d4422017-08-03 08:27:51 -07002994TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002995 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2996 const int kWidth = 1280;
2997 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002999 DataRate::BitsPerSec(kTargetBitrateBps),
3000 DataRate::BitsPerSec(kTargetBitrateBps),
3001 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003002
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003003 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003004 AdaptingFrameForwarder source;
3005 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003007 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003008
Åsa Persson8c1bf952018-09-13 10:42:19 +02003009 int64_t timestamp_ms = kFrameIntervalMs;
3010 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003011 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003012 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003013 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3014 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3015
3016 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003017 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003018 timestamp_ms += kFrameIntervalMs;
3019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3020 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003021 EXPECT_THAT(source.sink_wants(),
3022 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003023 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3024 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3025
3026 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003027 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003028 timestamp_ms += kFrameIntervalMs;
3029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003030 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003031 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003032 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3033 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3034
3035 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003037 timestamp_ms += kFrameIntervalMs;
3038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3039 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003040 EXPECT_THAT(source.sink_wants(),
3041 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003042 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3043 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3044
3045 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003046 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003047 timestamp_ms += kFrameIntervalMs;
3048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003049 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003051 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3052 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3053
mflodmancc3d4422017-08-03 08:27:51 -07003054 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003055}
3056
mflodmancc3d4422017-08-03 08:27:51 -07003057TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003058 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3059 const int kWidth = 1280;
3060 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003062 DataRate::BitsPerSec(kTargetBitrateBps),
3063 DataRate::BitsPerSec(kTargetBitrateBps),
3064 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003065
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003066 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003067 AdaptingFrameForwarder source;
3068 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003069 video_stream_encoder_->SetSource(&source,
3070 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003071
Åsa Persson8c1bf952018-09-13 10:42:19 +02003072 int64_t timestamp_ms = kFrameIntervalMs;
3073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003074 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003075 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003076 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3077 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3078
3079 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003080 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003081 timestamp_ms += kFrameIntervalMs;
3082 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3083 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003084 EXPECT_THAT(source.sink_wants(),
3085 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003086 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3087 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3088
3089 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003090 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003091 timestamp_ms += kFrameIntervalMs;
3092 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003093 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003094 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003095 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3096 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3097
3098 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003100 timestamp_ms += kFrameIntervalMs;
3101 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3102 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003103 EXPECT_THAT(source.sink_wants(),
3104 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003105 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3106 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3107
3108 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003109 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003110 timestamp_ms += kFrameIntervalMs;
3111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003112 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003113 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003114 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3115 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3116
mflodmancc3d4422017-08-03 08:27:51 -07003117 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003118}
3119
Sergey Silkin41c650b2019-10-14 13:12:19 +02003120TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3121 fake_encoder_.SetResolutionBitrateLimits(
3122 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3123
Henrik Boström381d1092020-05-12 18:49:07 +02003124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003125 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3126 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3127 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3128 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003129
3130 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3131 AdaptingFrameForwarder source;
3132 source.set_adaptation_enabled(true);
3133 video_stream_encoder_->SetSource(
3134 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3135
3136 // Insert 720p frame.
3137 int64_t timestamp_ms = kFrameIntervalMs;
3138 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3139 WaitForEncodedFrame(1280, 720);
3140
3141 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003142 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003143 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3144 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3145 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3146 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003147 video_stream_encoder_->TriggerQualityLow();
3148
3149 // Insert 720p frame. It should be downscaled and encoded.
3150 timestamp_ms += kFrameIntervalMs;
3151 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3152 WaitForEncodedFrame(960, 540);
3153
3154 // Trigger adapt up. Higher resolution should not be requested duo to lack
3155 // of bitrate.
3156 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003157 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003158
3159 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003161 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3162 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3163 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3164 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003165
3166 // Trigger adapt up. Higher resolution should be requested.
3167 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003168 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003169
3170 video_stream_encoder_->Stop();
3171}
3172
3173TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3174 fake_encoder_.SetResolutionBitrateLimits(
3175 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3176
3177 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003179 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3180 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3181 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3182 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003183
3184 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3185 AdaptingFrameForwarder source;
3186 source.set_adaptation_enabled(true);
3187 video_stream_encoder_->SetSource(
3188 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3189
3190 // Insert 720p frame. It should be dropped and lower resolution should be
3191 // requested.
3192 int64_t timestamp_ms = kFrameIntervalMs;
3193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3194 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003195 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003196
3197 // Insert 720p frame. It should be downscaled and encoded.
3198 timestamp_ms += kFrameIntervalMs;
3199 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3200 WaitForEncodedFrame(960, 540);
3201
3202 video_stream_encoder_->Stop();
3203}
3204
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003205class BalancedDegradationTest : public VideoStreamEncoderTest {
3206 protected:
3207 void SetupTest() {
3208 // Reset encoder for field trials to take effect.
3209 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003210 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003211
3212 // Enable BALANCED preference.
3213 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003214 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3215 }
3216
3217 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003219 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3220 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003221 }
3222
Åsa Persson45b176f2019-09-30 11:19:05 +02003223 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003224 timestamp_ms_ += kFrameIntervalMs;
3225 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003226 }
3227
3228 void InsertFrameAndWaitForEncoded() {
3229 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003230 sink_.WaitForEncodedFrame(timestamp_ms_);
3231 }
3232
3233 const int kWidth = 640; // pixels:640x360=230400
3234 const int kHeight = 360;
3235 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3236 int64_t timestamp_ms_ = 0;
3237 AdaptingFrameForwarder source_;
3238};
3239
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003240TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003241 test::ScopedFieldTrials field_trials(
3242 "WebRTC-Video-BalancedDegradationSettings/"
3243 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3244 SetupTest();
3245
3246 // Force input frame rate.
3247 const int kInputFps = 24;
3248 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3249 stats.input_frame_rate = kInputFps;
3250 stats_proxy_->SetMockStats(stats);
3251
Åsa Persson45b176f2019-09-30 11:19:05 +02003252 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003253 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003254
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003255 // Trigger adapt down, expect scaled down framerate and resolution,
3256 // since Fps diff (input-requested:0) < threshold.
3257 video_stream_encoder_->TriggerQualityLow();
3258 EXPECT_THAT(source_.sink_wants(),
3259 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003260
3261 video_stream_encoder_->Stop();
3262}
3263
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003264TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003265 test::ScopedFieldTrials field_trials(
3266 "WebRTC-Video-BalancedDegradationSettings/"
3267 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3268 SetupTest();
3269
3270 // Force input frame rate.
3271 const int kInputFps = 25;
3272 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3273 stats.input_frame_rate = kInputFps;
3274 stats_proxy_->SetMockStats(stats);
3275
Åsa Persson45b176f2019-09-30 11:19:05 +02003276 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003277 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003278
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003279 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3280 // Fps diff (input-requested:1) == threshold.
3281 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003282 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003283
3284 video_stream_encoder_->Stop();
3285}
3286
3287TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3288 test::ScopedFieldTrials field_trials(
3289 "WebRTC-Video-BalancedDegradationSettings/"
3290 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3291 SetupTest();
3292
3293 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3294
Åsa Persson45b176f2019-09-30 11:19:05 +02003295 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003296 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003297
3298 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3299 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003300 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003301
3302 video_stream_encoder_->Stop();
3303}
3304
Åsa Perssonccfb3402019-09-25 15:13:04 +02003305TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003306 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003307 "WebRTC-Video-BalancedDegradationSettings/"
3308 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003309 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003310
Åsa Persson1b247f12019-08-14 17:26:39 +02003311 const int kMinBitrateBps = 425000;
3312 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003313 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003314
Åsa Persson45b176f2019-09-30 11:19:05 +02003315 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003316 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003317 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3318
3319 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3320 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003321 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003323 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3324
3325 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3326 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003327 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003328 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003329 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3330
Åsa Persson30ab0152019-08-27 12:22:33 +02003331 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3332 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003333 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003334 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003335 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003336 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3337
3338 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003340 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003341 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003342
Åsa Persson30ab0152019-08-27 12:22:33 +02003343 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003344 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003345 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003346 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003347 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003348 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3349
3350 video_stream_encoder_->Stop();
3351}
3352
Åsa Perssonccfb3402019-09-25 15:13:04 +02003353TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003354 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3355 test::ScopedFieldTrials field_trials(
3356 "WebRTC-Video-BalancedDegradationSettings/"
3357 "pixels:57600|129600|230400,fps:7|24|24/");
3358 SetupTest();
3359 OnBitrateUpdated(kLowTargetBitrateBps);
3360
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003361 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003362
3363 // Insert frame, expect scaled down:
3364 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3365 InsertFrame();
3366 EXPECT_FALSE(WaitForFrame(1000));
3367 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3368 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3369
3370 // Insert frame, expect scaled down:
3371 // resolution (320x180@24fps).
3372 InsertFrame();
3373 EXPECT_FALSE(WaitForFrame(1000));
3374 EXPECT_LT(source_.sink_wants().max_pixel_count,
3375 source_.last_wants().max_pixel_count);
3376 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3377
3378 // Frame should not be dropped (min pixels per frame reached).
3379 InsertFrameAndWaitForEncoded();
3380
3381 video_stream_encoder_->Stop();
3382}
3383
3384TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003385 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003386 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003387 "WebRTC-Video-BalancedDegradationSettings/"
3388 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003389 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003390
Åsa Persson30ab0152019-08-27 12:22:33 +02003391 const int kResolutionMinBitrateBps = 435000;
3392 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003393 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003394
Åsa Persson45b176f2019-09-30 11:19:05 +02003395 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003396 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003397 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3398
3399 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3400 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003401 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003402 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003403 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3404
3405 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3406 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003407 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003408 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003409 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3410
3411 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3412 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003413 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003414 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003415 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3416
Åsa Persson30ab0152019-08-27 12:22:33 +02003417 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3418 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003419 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003420 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003421 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3422
3423 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3424 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003425 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003426 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3427
3428 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003429 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003430 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003431 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003432 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003433 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3434
3435 video_stream_encoder_->Stop();
3436}
3437
Åsa Perssonccfb3402019-09-25 15:13:04 +02003438TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003439 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003440 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003441 "WebRTC-Video-BalancedDegradationSettings/"
3442 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003443 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003444
Åsa Persson30ab0152019-08-27 12:22:33 +02003445 const int kMinBitrateBps = 425000;
3446 const int kTooLowMinBitrateBps = 424000;
3447 const int kResolutionMinBitrateBps = 435000;
3448 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003449 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003450
Åsa Persson45b176f2019-09-30 11:19:05 +02003451 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003452 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003453 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3454
3455 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3456 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003457 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003458 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003459 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3462 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003463 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003464 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003465 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3466
3467 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3468 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003469 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003470 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003471 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3474 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003475 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003476 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3477
3478 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003479 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003480 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003481 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003482 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003483 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3484
3485 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003486 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003487 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003488 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003489 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3490
3491 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003492 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003493 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003494 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003495 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003496 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3497
Åsa Persson1b247f12019-08-14 17:26:39 +02003498 video_stream_encoder_->Stop();
3499}
3500
mflodmancc3d4422017-08-03 08:27:51 -07003501TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003502 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3503 const int kWidth = 1280;
3504 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003506 DataRate::BitsPerSec(kTargetBitrateBps),
3507 DataRate::BitsPerSec(kTargetBitrateBps),
3508 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003509
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003510 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003511 AdaptingFrameForwarder source;
3512 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003513 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003514 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003515
Åsa Persson8c1bf952018-09-13 10:42:19 +02003516 int64_t timestamp_ms = kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003518 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003519 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003520 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3521 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3522 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3523 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3524
3525 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003526 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003527 timestamp_ms += kFrameIntervalMs;
3528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3529 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003530 EXPECT_THAT(source.sink_wants(),
3531 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003532 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3533 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3535 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3536
3537 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003538 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003539 timestamp_ms += kFrameIntervalMs;
3540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3541 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003542 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003543 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3544 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3545 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3546 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3547
Jonathan Yubc771b72017-12-08 17:04:29 -08003548 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003549 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003550 timestamp_ms += kFrameIntervalMs;
3551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003553 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003554 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003556 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003557 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3558
Jonathan Yubc771b72017-12-08 17:04:29 -08003559 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003560 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003561 timestamp_ms += kFrameIntervalMs;
3562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003565 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003566 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3568 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3569 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3570
Jonathan Yubc771b72017-12-08 17:04:29 -08003571 // Trigger quality adapt down, expect no change (min resolution reached).
3572 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003573 timestamp_ms += kFrameIntervalMs;
3574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3575 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003576 EXPECT_THAT(source.sink_wants(), FpsMax());
3577 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003578 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3581 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3582
Evan Shrubsole64469032020-06-11 10:45:29 +02003583 // Trigger quality adapt up, expect upscaled resolution (480x270).
3584 video_stream_encoder_->TriggerQualityHigh();
3585 timestamp_ms += kFrameIntervalMs;
3586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3587 WaitForEncodedFrame(timestamp_ms);
3588 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3589 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3591 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3592 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3593
3594 // Trigger quality and cpu adapt up since both are most limited, expect
3595 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003596 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003597 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003598 timestamp_ms += kFrameIntervalMs;
3599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3600 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003601 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003602 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3604 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003605 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003606
Evan Shrubsole64469032020-06-11 10:45:29 +02003607 // Trigger quality and cpu adapt up since both are most limited, expect
3608 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003609 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003610 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003611 timestamp_ms += kFrameIntervalMs;
3612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3613 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003614 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003615 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003618 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3619 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003620
Evan Shrubsole64469032020-06-11 10:45:29 +02003621 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3622 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003623 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003624 timestamp_ms += kFrameIntervalMs;
3625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3626 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003627 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003630 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003631 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003632
3633 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003634 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003635 timestamp_ms += kFrameIntervalMs;
3636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003637 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003638 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003639 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003642 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003643 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003644
mflodmancc3d4422017-08-03 08:27:51 -07003645 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003646}
3647
mflodmancc3d4422017-08-03 08:27:51 -07003648TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003649 const int kWidth = 640;
3650 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003651
Henrik Boström381d1092020-05-12 18:49:07 +02003652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003653 DataRate::BitsPerSec(kTargetBitrateBps),
3654 DataRate::BitsPerSec(kTargetBitrateBps),
3655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003656
perkj803d97f2016-11-01 11:45:46 -07003657 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003658 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003659 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003660 }
3661
mflodmancc3d4422017-08-03 08:27:51 -07003662 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003663 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003664 video_source_.IncomingCapturedFrame(CreateFrame(
3665 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003666 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003667 }
3668
mflodmancc3d4422017-08-03 08:27:51 -07003669 video_stream_encoder_->Stop();
3670 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003671 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003672
Ying Wangef3998f2019-12-09 13:06:53 +01003673 EXPECT_METRIC_EQ(
3674 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3675 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003676 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3677}
3678
mflodmancc3d4422017-08-03 08:27:51 -07003679TEST_F(VideoStreamEncoderTest,
3680 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003682 DataRate::BitsPerSec(kTargetBitrateBps),
3683 DataRate::BitsPerSec(kTargetBitrateBps),
3684 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003685 const int kWidth = 640;
3686 const int kHeight = 360;
3687
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003688 video_stream_encoder_->SetSource(&video_source_,
3689 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003690
3691 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3692 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003693 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003694 }
3695
mflodmancc3d4422017-08-03 08:27:51 -07003696 video_stream_encoder_->Stop();
3697 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003698 stats_proxy_.reset();
3699
3700 EXPECT_EQ(0,
3701 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3702}
3703
mflodmancc3d4422017-08-03 08:27:51 -07003704TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003705 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003706 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003707
3708 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003709 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003710 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003711 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3712 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003713
sprang57c2fff2017-01-16 06:24:02 -08003714 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003715 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003717 DataRate::BitsPerSec(kLowTargetBitrateBps),
3718 DataRate::BitsPerSec(kLowTargetBitrateBps),
3719 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003720
sprang57c2fff2017-01-16 06:24:02 -08003721 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003722 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3723 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003724 VideoBitrateAllocation bitrate_allocation =
3725 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003726 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003727 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003728 // TODO(srte): The use of millisecs here looks like an error, but the tests
3729 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003730 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003731
3732 // Not called on second frame.
3733 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3734 .Times(0);
3735 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003736 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3737 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003738 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003739
3740 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003741 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3742 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003743 const int64_t start_time_ms = rtc::TimeMillis();
3744 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3745 video_source_.IncomingCapturedFrame(
3746 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3747 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003748 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003749 }
3750
3751 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003752 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003753
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003755}
3756
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003757TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3758 // 2 TLs configured, temporal layers supported by encoder.
3759 const int kNumTemporalLayers = 2;
3760 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3761 fake_encoder_.SetTemporalLayersSupported(0, true);
3762
3763 // Bitrate allocated across temporal layers.
3764 const int kTl0Bps = kTargetBitrateBps *
3765 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003766 kNumTemporalLayers, /*temporal_id*/ 0,
3767 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003768 const int kTl1Bps = kTargetBitrateBps *
3769 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003770 kNumTemporalLayers, /*temporal_id*/ 1,
3771 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003772 VideoBitrateAllocation expected_bitrate;
3773 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3774 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3775
3776 VerifyAllocatedBitrate(expected_bitrate);
3777 video_stream_encoder_->Stop();
3778}
3779
3780TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3781 // 2 TLs configured, temporal layers not supported by encoder.
3782 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3783 fake_encoder_.SetTemporalLayersSupported(0, false);
3784
3785 // Temporal layers not supported by the encoder.
3786 // Total bitrate should be at ti:0.
3787 VideoBitrateAllocation expected_bitrate;
3788 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3789
3790 VerifyAllocatedBitrate(expected_bitrate);
3791 video_stream_encoder_->Stop();
3792}
3793
3794TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3795 // 2 TLs configured, temporal layers only supported for first stream.
3796 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3797 fake_encoder_.SetTemporalLayersSupported(0, true);
3798 fake_encoder_.SetTemporalLayersSupported(1, false);
3799
3800 const int kS0Bps = 150000;
3801 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003802 kS0Bps *
3803 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3804 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003805 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003806 kS0Bps *
3807 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3808 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003809 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3810 // Temporal layers not supported by si:1.
3811 VideoBitrateAllocation expected_bitrate;
3812 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3813 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3814 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3815
3816 VerifyAllocatedBitrate(expected_bitrate);
3817 video_stream_encoder_->Stop();
3818}
3819
Niels Möller7dc26b72017-12-06 10:27:48 +01003820TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3821 const int kFrameWidth = 1280;
3822 const int kFrameHeight = 720;
3823 const int kFramerate = 24;
3824
Henrik Boström381d1092020-05-12 18:49:07 +02003825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003826 DataRate::BitsPerSec(kTargetBitrateBps),
3827 DataRate::BitsPerSec(kTargetBitrateBps),
3828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003829 test::FrameForwarder source;
3830 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003831 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003832
3833 // Insert a single frame, triggering initial configuration.
3834 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3836
3837 EXPECT_EQ(
3838 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3839 kDefaultFramerate);
3840
3841 // Trigger reconfigure encoder (without resetting the entire instance).
3842 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003843 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003844 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3845 video_encoder_config.number_of_streams = 1;
3846 video_encoder_config.video_stream_factory =
3847 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3848 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003849 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003850 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3851
3852 // Detector should be updated with fps limit from codec config.
3853 EXPECT_EQ(
3854 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3855 kFramerate);
3856
3857 // Trigger overuse, max framerate should be reduced.
3858 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3859 stats.input_frame_rate = kFramerate;
3860 stats_proxy_->SetMockStats(stats);
3861 video_stream_encoder_->TriggerCpuOveruse();
3862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3863 int adapted_framerate =
3864 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3865 EXPECT_LT(adapted_framerate, kFramerate);
3866
3867 // Trigger underuse, max framerate should go back to codec configured fps.
3868 // Set extra low fps, to make sure it's actually reset, not just incremented.
3869 stats = stats_proxy_->GetStats();
3870 stats.input_frame_rate = adapted_framerate / 2;
3871 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003872 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3874 EXPECT_EQ(
3875 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3876 kFramerate);
3877
3878 video_stream_encoder_->Stop();
3879}
3880
3881TEST_F(VideoStreamEncoderTest,
3882 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3883 const int kFrameWidth = 1280;
3884 const int kFrameHeight = 720;
3885 const int kLowFramerate = 15;
3886 const int kHighFramerate = 25;
3887
Henrik Boström381d1092020-05-12 18:49:07 +02003888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003889 DataRate::BitsPerSec(kTargetBitrateBps),
3890 DataRate::BitsPerSec(kTargetBitrateBps),
3891 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003892 test::FrameForwarder source;
3893 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003894 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003895
3896 // Trigger initial configuration.
3897 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003898 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003899 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3900 video_encoder_config.number_of_streams = 1;
3901 video_encoder_config.video_stream_factory =
3902 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3903 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3904 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003905 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003906 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3907
3908 EXPECT_EQ(
3909 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3910 kLowFramerate);
3911
3912 // Trigger overuse, max framerate should be reduced.
3913 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3914 stats.input_frame_rate = kLowFramerate;
3915 stats_proxy_->SetMockStats(stats);
3916 video_stream_encoder_->TriggerCpuOveruse();
3917 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3918 int adapted_framerate =
3919 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3920 EXPECT_LT(adapted_framerate, kLowFramerate);
3921
3922 // Reconfigure the encoder with a new (higher max framerate), max fps should
3923 // still respect the adaptation.
3924 video_encoder_config.video_stream_factory =
3925 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3926 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3927 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003928 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003929 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3930
3931 EXPECT_EQ(
3932 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3933 adapted_framerate);
3934
3935 // Trigger underuse, max framerate should go back to codec configured fps.
3936 stats = stats_proxy_->GetStats();
3937 stats.input_frame_rate = adapted_framerate;
3938 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003939 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003940 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3941 EXPECT_EQ(
3942 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3943 kHighFramerate);
3944
3945 video_stream_encoder_->Stop();
3946}
3947
mflodmancc3d4422017-08-03 08:27:51 -07003948TEST_F(VideoStreamEncoderTest,
3949 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003950 const int kFrameWidth = 1280;
3951 const int kFrameHeight = 720;
3952 const int kFramerate = 24;
3953
Henrik Boström381d1092020-05-12 18:49:07 +02003954 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003955 DataRate::BitsPerSec(kTargetBitrateBps),
3956 DataRate::BitsPerSec(kTargetBitrateBps),
3957 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003958 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003959 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003960 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003961
3962 // Trigger initial configuration.
3963 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003964 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003965 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3966 video_encoder_config.number_of_streams = 1;
3967 video_encoder_config.video_stream_factory =
3968 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3969 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003970 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003971 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003972 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003973
Niels Möller7dc26b72017-12-06 10:27:48 +01003974 EXPECT_EQ(
3975 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3976 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003977
3978 // Trigger overuse, max framerate should be reduced.
3979 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3980 stats.input_frame_rate = kFramerate;
3981 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003982 video_stream_encoder_->TriggerCpuOveruse();
3983 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003984 int adapted_framerate =
3985 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003986 EXPECT_LT(adapted_framerate, kFramerate);
3987
3988 // Change degradation preference to not enable framerate scaling. Target
3989 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02003990 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003991 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01003992 EXPECT_EQ(
3993 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3994 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003995
mflodmancc3d4422017-08-03 08:27:51 -07003996 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003997}
3998
mflodmancc3d4422017-08-03 08:27:51 -07003999TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004000 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004002 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4003 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4004 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004005 const int kWidth = 640;
4006 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004007
asaperssonfab67072017-04-04 05:51:49 -07004008 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004009
4010 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004011 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004012
4013 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004014 EXPECT_TRUE_WAIT(
4015 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004016
sprangc5d62e22017-04-02 23:53:04 -07004017 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004018
asaperssonfab67072017-04-04 05:51:49 -07004019 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004020 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004021 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004022
4023 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004024 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004025
Henrik Boström2671dac2020-05-19 16:29:09 +02004026 EXPECT_TRUE_WAIT(
4027 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004028
mflodmancc3d4422017-08-03 08:27:51 -07004029 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004030}
4031
mflodmancc3d4422017-08-03 08:27:51 -07004032TEST_F(VideoStreamEncoderTest,
4033 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004034 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004036 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4037 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4038 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004039 const int kWidth = 640;
4040 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004041
4042 // We expect the n initial frames to get dropped.
4043 int i;
4044 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004045 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004046 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004047 }
4048 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004049 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004050 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004051
4052 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004053 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004054
mflodmancc3d4422017-08-03 08:27:51 -07004055 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004056}
4057
mflodmancc3d4422017-08-03 08:27:51 -07004058TEST_F(VideoStreamEncoderTest,
4059 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004060 const int kWidth = 640;
4061 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004063 DataRate::BitsPerSec(kLowTargetBitrateBps),
4064 DataRate::BitsPerSec(kLowTargetBitrateBps),
4065 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004066
4067 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004068 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004069 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004070
asaperssonfab67072017-04-04 05:51:49 -07004071 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004072 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004073 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004074
mflodmancc3d4422017-08-03 08:27:51 -07004075 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004076}
4077
mflodmancc3d4422017-08-03 08:27:51 -07004078TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004079 const int kWidth = 640;
4080 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004081 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004082
4083 VideoEncoderConfig video_encoder_config;
4084 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4085 // Make format different, to force recreation of encoder.
4086 video_encoder_config.video_format.parameters["foo"] = "foo";
4087 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004088 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004090 DataRate::BitsPerSec(kLowTargetBitrateBps),
4091 DataRate::BitsPerSec(kLowTargetBitrateBps),
4092 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004093
kthelgasonb83797b2017-02-14 11:57:25 -08004094 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004095 video_stream_encoder_->SetSource(&video_source_,
4096 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004097
asaperssonfab67072017-04-04 05:51:49 -07004098 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004099 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004100 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004101
mflodmancc3d4422017-08-03 08:27:51 -07004102 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004103 fake_encoder_.SetQualityScaling(true);
4104}
4105
Åsa Persson139f4dc2019-08-02 09:29:58 +02004106TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4107 webrtc::test::ScopedFieldTrials field_trials(
4108 "WebRTC-Video-QualityScalerSettings/"
4109 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4110 // Reset encoder for field trials to take effect.
4111 ConfigureEncoder(video_encoder_config_.Copy());
4112 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4113 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4114 const int kWidth = 640;
4115 const int kHeight = 360;
4116
Henrik Boström381d1092020-05-12 18:49:07 +02004117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004118 DataRate::BitsPerSec(kTargetBitrateBps),
4119 DataRate::BitsPerSec(kTargetBitrateBps),
4120 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004121 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4122 // Frame should not be dropped.
4123 WaitForEncodedFrame(1);
4124
Henrik Boström381d1092020-05-12 18:49:07 +02004125 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004126 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4127 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4128 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004129 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4130 // Frame should not be dropped.
4131 WaitForEncodedFrame(2);
4132
Henrik Boström381d1092020-05-12 18:49:07 +02004133 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004134 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4135 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4136 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004137 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4138 // Expect to drop this frame, the wait should time out.
4139 ExpectDroppedFrame();
4140
4141 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004142 EXPECT_TRUE_WAIT(
4143 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004144 video_stream_encoder_->Stop();
4145}
4146
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02004147TEST_F(VideoStreamEncoderTest,
4148 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
4149 webrtc::test::ScopedFieldTrials field_trials(
4150 "WebRTC-Video-QualityScalerSettings/"
4151 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4152 fake_encoder_.SetQualityScaling(false);
4153 ConfigureEncoder(video_encoder_config_.Copy());
4154 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4155 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4156 const int kWidth = 640;
4157 const int kHeight = 360;
4158
4159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4160 DataRate::BitsPerSec(kTargetBitrateBps),
4161 DataRate::BitsPerSec(kTargetBitrateBps),
4162 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4163 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4164 // Frame should not be dropped.
4165 WaitForEncodedFrame(1);
4166
4167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4168 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4169 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4170 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
4171 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4172 // Frame should not be dropped.
4173 WaitForEncodedFrame(2);
4174
4175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4176 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4177 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4178 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
4179 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4180 // Not dropped since quality scaling is disabled.
4181 WaitForEncodedFrame(3);
4182
4183 // Expect the sink_wants to specify a scaled frame.
4184 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
4185 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
4186
4187 video_stream_encoder_->Stop();
4188}
4189
Åsa Perssone644a032019-11-08 15:56:00 +01004190TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4191 webrtc::test::ScopedFieldTrials field_trials(
4192 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4193
4194 // Reset encoder for field trials to take effect.
4195 VideoEncoderConfig config = video_encoder_config_.Copy();
4196 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004197 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004198 ConfigureEncoder(std::move(config));
4199 fake_encoder_.SetQp(kQpLow);
4200
4201 // Enable MAINTAIN_FRAMERATE preference.
4202 AdaptingFrameForwarder source;
4203 source.set_adaptation_enabled(true);
4204 video_stream_encoder_->SetSource(&source,
4205 DegradationPreference::MAINTAIN_FRAMERATE);
4206
4207 // Start at low bitrate.
4208 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4210 DataRate::BitsPerSec(kLowBitrateBps),
4211 DataRate::BitsPerSec(kLowBitrateBps),
4212 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004213
4214 // Expect first frame to be dropped and resolution to be limited.
4215 const int kWidth = 1280;
4216 const int kHeight = 720;
4217 const int64_t kFrameIntervalMs = 100;
4218 int64_t timestamp_ms = kFrameIntervalMs;
4219 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4220 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004221 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4222 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004223
4224 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004225 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4226 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004227
4228 // Insert frames and advance |min_duration_ms|.
4229 for (size_t i = 1; i <= 10; i++) {
4230 timestamp_ms += kFrameIntervalMs;
4231 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4232 WaitForEncodedFrame(timestamp_ms);
4233 }
4234 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4235 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4236
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004237 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004238
4239 // Insert frame should trigger high BW and release quality limitation.
4240 timestamp_ms += kFrameIntervalMs;
4241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4242 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004243 // The ramp-up code involves the adaptation queue, give it time to execute.
4244 // TODO(hbos): Can we await an appropriate event instead?
4245 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004246 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004247
4248 // Frame should not be adapted.
4249 timestamp_ms += kFrameIntervalMs;
4250 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4251 WaitForEncodedFrame(kWidth, kHeight);
4252 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4253
4254 video_stream_encoder_->Stop();
4255}
4256
mflodmancc3d4422017-08-03 08:27:51 -07004257TEST_F(VideoStreamEncoderTest,
Evan Shrubsoleba8abbb2020-08-13 09:34:09 +02004258 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
4259 AdaptingFrameForwarder source;
4260 source.set_adaptation_enabled(true);
4261 video_stream_encoder_->SetSource(&source,
4262 DegradationPreference::MAINTAIN_FRAMERATE);
4263 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4264 DataRate::BitsPerSec(kTargetBitrateBps),
4265 DataRate::BitsPerSec(kTargetBitrateBps),
4266 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4267 fake_encoder_.SetQp(kQpHigh + 1);
4268 const int kWidth = 1280;
4269 const int kHeight = 720;
4270 const int64_t kFrameIntervalMs = 100;
4271 int64_t timestamp_ms = kFrameIntervalMs;
4272 for (size_t i = 1; i <= 100; i++) {
4273 timestamp_ms += kFrameIntervalMs;
4274 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4275 WaitForEncodedFrame(timestamp_ms);
4276 }
4277 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
4278 // for the first time.
4279 // TODO(eshr): We should avoid these waits by using threads with simulated
4280 // time.
4281 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
4282 2000 * 2.5 * 2);
4283 timestamp_ms += kFrameIntervalMs;
4284 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4285 WaitForEncodedFrame(timestamp_ms);
4286 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4287 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
4288 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
4289 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4290
4291 // Disable Quality scaling by turning off scaler on the encoder and
4292 // reconfiguring.
4293 fake_encoder_.SetQualityScaling(false);
4294 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
4295 kMaxPayloadLength);
4296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4297 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
4298 // Since we turned off the quality scaler, the adaptations made by it are
4299 // removed.
4300 EXPECT_THAT(source.sink_wants(), ResolutionMax());
4301 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4302
4303 video_stream_encoder_->Stop();
4304}
4305
4306TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004307 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4308 const int kTooSmallWidth = 10;
4309 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004311 DataRate::BitsPerSec(kTargetBitrateBps),
4312 DataRate::BitsPerSec(kTargetBitrateBps),
4313 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004314
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004315 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004316 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004317 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004318 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004319 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004320 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4321
4322 // Trigger adapt down, too small frame, expect no change.
4323 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004324 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004325 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004326 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004327 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4328 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4329
mflodmancc3d4422017-08-03 08:27:51 -07004330 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004331}
4332
mflodmancc3d4422017-08-03 08:27:51 -07004333TEST_F(VideoStreamEncoderTest,
4334 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004335 const int kTooSmallWidth = 10;
4336 const int kTooSmallHeight = 10;
4337 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004339 DataRate::BitsPerSec(kTargetBitrateBps),
4340 DataRate::BitsPerSec(kTargetBitrateBps),
4341 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004342
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004343 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004344 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004345 video_stream_encoder_->SetSource(&source,
4346 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004347 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004348 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4349 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4350
4351 // Trigger adapt down, expect limited framerate.
4352 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004353 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004354 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004355 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004356 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4358 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4359
4360 // Trigger adapt down, too small frame, expect no change.
4361 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004362 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004363 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004364 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4366 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4367 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4368
mflodmancc3d4422017-08-03 08:27:51 -07004369 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004370}
4371
mflodmancc3d4422017-08-03 08:27:51 -07004372TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004373 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004374 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004375 DataRate::BitsPerSec(kTargetBitrateBps),
4376 DataRate::BitsPerSec(kTargetBitrateBps),
4377 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004378 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004379 const int kFrameWidth = 1280;
4380 const int kFrameHeight = 720;
4381 video_source_.IncomingCapturedFrame(
4382 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004383 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004384 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004385}
4386
sprangb1ca0732017-02-01 08:38:12 -08004387// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004388TEST_F(VideoStreamEncoderTest,
4389 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004391 DataRate::BitsPerSec(kTargetBitrateBps),
4392 DataRate::BitsPerSec(kTargetBitrateBps),
4393 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004394
4395 const int kFrameWidth = 1280;
4396 const int kFrameHeight = 720;
4397 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004398 // requested by
4399 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004400 video_source_.set_adaptation_enabled(true);
4401
4402 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004403 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004404 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004405
4406 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004407 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004408 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004409 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004410 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004411
asaperssonfab67072017-04-04 05:51:49 -07004412 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004413 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004414 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004415 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004416 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004417
mflodmancc3d4422017-08-03 08:27:51 -07004418 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004419}
sprangfe627f32017-03-29 08:24:59 -07004420
mflodmancc3d4422017-08-03 08:27:51 -07004421TEST_F(VideoStreamEncoderTest,
4422 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004423 const int kFrameWidth = 1280;
4424 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004425
Henrik Boström381d1092020-05-12 18:49:07 +02004426 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004427 DataRate::BitsPerSec(kTargetBitrateBps),
4428 DataRate::BitsPerSec(kTargetBitrateBps),
4429 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004430 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004431 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004432 video_source_.set_adaptation_enabled(true);
4433
sprang4847ae62017-06-27 07:06:52 -07004434 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004435
4436 video_source_.IncomingCapturedFrame(
4437 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004438 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004439
4440 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004441 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004442
4443 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004444 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004445 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004446 video_source_.IncomingCapturedFrame(
4447 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004448 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004449 }
4450
4451 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004452 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004453 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004454 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004455 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004456 video_source_.IncomingCapturedFrame(
4457 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004458 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004459 ++num_frames_dropped;
4460 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004461 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004462 }
4463 }
4464
sprang4847ae62017-06-27 07:06:52 -07004465 // Add some slack to account for frames dropped by the frame dropper.
4466 const int kErrorMargin = 1;
4467 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004468 kErrorMargin);
4469
4470 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004471 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004472 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004473 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004474 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004475 video_source_.IncomingCapturedFrame(
4476 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004477 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004478 ++num_frames_dropped;
4479 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004480 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004481 }
4482 }
sprang4847ae62017-06-27 07:06:52 -07004483 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004484 kErrorMargin);
4485
4486 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004487 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004488 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004489 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004490 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004491 video_source_.IncomingCapturedFrame(
4492 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004493 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004494 ++num_frames_dropped;
4495 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004496 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004497 }
4498 }
sprang4847ae62017-06-27 07:06:52 -07004499 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004500 kErrorMargin);
4501
4502 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004503 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004504 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004505 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004506 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004507 video_source_.IncomingCapturedFrame(
4508 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004509 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004510 ++num_frames_dropped;
4511 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004512 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004513 }
4514 }
4515 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4516
mflodmancc3d4422017-08-03 08:27:51 -07004517 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004518}
4519
mflodmancc3d4422017-08-03 08:27:51 -07004520TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004521 const int kFramerateFps = 5;
4522 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004523 const int kFrameWidth = 1280;
4524 const int kFrameHeight = 720;
4525
sprang4847ae62017-06-27 07:06:52 -07004526 // Reconfigure encoder with two temporal layers and screensharing, which will
4527 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004528 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004529
Henrik Boström381d1092020-05-12 18:49:07 +02004530 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004531 DataRate::BitsPerSec(kTargetBitrateBps),
4532 DataRate::BitsPerSec(kTargetBitrateBps),
4533 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004534 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004535 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004536 video_source_.set_adaptation_enabled(true);
4537
sprang4847ae62017-06-27 07:06:52 -07004538 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004539
4540 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004541 rtc::VideoSinkWants last_wants;
4542 do {
4543 last_wants = video_source_.sink_wants();
4544
sprangc5d62e22017-04-02 23:53:04 -07004545 // Insert frames to get a new fps estimate...
4546 for (int j = 0; j < kFramerateFps; ++j) {
4547 video_source_.IncomingCapturedFrame(
4548 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004549 if (video_source_.last_sent_width()) {
4550 sink_.WaitForEncodedFrame(timestamp_ms);
4551 }
sprangc5d62e22017-04-02 23:53:04 -07004552 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004553 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004554 }
4555 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004556 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004557 } while (video_source_.sink_wants().max_framerate_fps <
4558 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004559
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004560 EXPECT_THAT(video_source_.sink_wants(),
4561 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004562
mflodmancc3d4422017-08-03 08:27:51 -07004563 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004564}
asaperssonf7e294d2017-06-13 23:25:22 -07004565
mflodmancc3d4422017-08-03 08:27:51 -07004566TEST_F(VideoStreamEncoderTest,
4567 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004568 const int kWidth = 1280;
4569 const int kHeight = 720;
4570 const int64_t kFrameIntervalMs = 150;
4571 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004573 DataRate::BitsPerSec(kTargetBitrateBps),
4574 DataRate::BitsPerSec(kTargetBitrateBps),
4575 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004576
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004577 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004578 AdaptingFrameForwarder source;
4579 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004580 video_stream_encoder_->SetSource(&source,
4581 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004582 timestamp_ms += kFrameIntervalMs;
4583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004584 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004585 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004586 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4587 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4588 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4589
4590 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004591 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004592 timestamp_ms += kFrameIntervalMs;
4593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004594 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004595 EXPECT_THAT(source.sink_wants(),
4596 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4598 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4599 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4600
4601 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004602 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004603 timestamp_ms += kFrameIntervalMs;
4604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004605 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004606 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4608 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4609 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4610
4611 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004612 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004613 timestamp_ms += kFrameIntervalMs;
4614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004616 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4619 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4620
4621 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004622 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004623 timestamp_ms += kFrameIntervalMs;
4624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004625 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004626 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4629 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4630
4631 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004632 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004633 timestamp_ms += kFrameIntervalMs;
4634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004635 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004636 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4638 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4639 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4640
4641 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004642 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004643 timestamp_ms += kFrameIntervalMs;
4644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004645 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004646 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4649 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4650
4651 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004652 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004653 timestamp_ms += kFrameIntervalMs;
4654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004655 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004656 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004657 rtc::VideoSinkWants last_wants = source.sink_wants();
4658 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4659 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4660 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4661
4662 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004663 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004664 timestamp_ms += kFrameIntervalMs;
4665 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004666 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004667 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004668 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4669 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4670 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4671
Evan Shrubsole64469032020-06-11 10:45:29 +02004672 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004673 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004674 timestamp_ms += kFrameIntervalMs;
4675 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004676 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004677 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004678 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4679 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4680 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4681
4682 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004683 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004684 timestamp_ms += kFrameIntervalMs;
4685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004686 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004687 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004688 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4689 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4690 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4691
4692 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004693 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004694 timestamp_ms += kFrameIntervalMs;
4695 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004696 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004697 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004698 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4699 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4700 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4701
4702 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004703 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004704 timestamp_ms += kFrameIntervalMs;
4705 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004706 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004707 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004708 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4709 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4710 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4711
4712 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004713 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004714 timestamp_ms += kFrameIntervalMs;
4715 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004716 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004717 EXPECT_THAT(source.sink_wants(), FpsMax());
4718 EXPECT_EQ(source.sink_wants().max_pixel_count,
4719 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004720 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4721 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4722 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4723
4724 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004725 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004726 timestamp_ms += kFrameIntervalMs;
4727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004728 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004730 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4732 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4733
Åsa Persson30ab0152019-08-27 12:22:33 +02004734 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004735 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004736 timestamp_ms += kFrameIntervalMs;
4737 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004738 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004739 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004740 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004741 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4743 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4744
4745 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004746 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004747 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004748 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4749
mflodmancc3d4422017-08-03 08:27:51 -07004750 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004751}
4752
mflodmancc3d4422017-08-03 08:27:51 -07004753TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004754 const int kWidth = 1280;
4755 const int kHeight = 720;
4756 const int64_t kFrameIntervalMs = 150;
4757 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004759 DataRate::BitsPerSec(kTargetBitrateBps),
4760 DataRate::BitsPerSec(kTargetBitrateBps),
4761 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004762
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004763 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004764 AdaptingFrameForwarder source;
4765 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004766 video_stream_encoder_->SetSource(&source,
4767 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004768 timestamp_ms += kFrameIntervalMs;
4769 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004770 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004771 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004772 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4774 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4775 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4776 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4777 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4778
4779 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004780 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004781 timestamp_ms += kFrameIntervalMs;
4782 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004783 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004784 EXPECT_THAT(source.sink_wants(),
4785 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004786 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4787 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4788 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4789 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4790 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4791 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4792
4793 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004794 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004795 timestamp_ms += kFrameIntervalMs;
4796 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004797 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004799 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4800 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4801 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4803 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4804 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4805
4806 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004807 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004808 timestamp_ms += kFrameIntervalMs;
4809 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004810 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004811 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02004812 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07004813 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4814 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4816 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4817 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4818
Evan Shrubsole64469032020-06-11 10:45:29 +02004819 // Trigger cpu adapt up, expect no change since QP is most limited.
4820 {
4821 // Store current sink wants since we expect no change and if there is no
4822 // change then last_wants() is not updated.
4823 auto previous_sink_wants = source.sink_wants();
4824 video_stream_encoder_->TriggerCpuUnderuse();
4825 timestamp_ms += kFrameIntervalMs;
4826 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4827 WaitForEncodedFrame(timestamp_ms);
4828 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4829 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4830 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4831 }
4832
4833 // Trigger quality adapt up, expect increased fps (640x360@30fps).
4834 video_stream_encoder_->TriggerQualityHigh();
4835 timestamp_ms += kFrameIntervalMs;
4836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4837 WaitForEncodedFrame(timestamp_ms);
4838 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
4839 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4841 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4843 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4844 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4845
4846 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4847 // expect increased resolution (960x540@30fps).
4848 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004849 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004850 timestamp_ms += kFrameIntervalMs;
4851 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004852 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02004853 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004854 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4856 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4857 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4858 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004859 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004860
Evan Shrubsole64469032020-06-11 10:45:29 +02004861 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4862 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004863 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004864 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004865 timestamp_ms += kFrameIntervalMs;
4866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004867 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004868 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004869 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004870 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4871 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4872 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4873 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4874 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004875 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004876
4877 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004878 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004879 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004880 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004881 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004882
mflodmancc3d4422017-08-03 08:27:51 -07004883 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004884}
4885
mflodmancc3d4422017-08-03 08:27:51 -07004886TEST_F(VideoStreamEncoderTest,
4887 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004888 const int kWidth = 640;
4889 const int kHeight = 360;
4890 const int kFpsLimit = 15;
4891 const int64_t kFrameIntervalMs = 150;
4892 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004894 DataRate::BitsPerSec(kTargetBitrateBps),
4895 DataRate::BitsPerSec(kTargetBitrateBps),
4896 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004897
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004898 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004899 AdaptingFrameForwarder source;
4900 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004901 video_stream_encoder_->SetSource(&source,
4902 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004903 timestamp_ms += kFrameIntervalMs;
4904 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004905 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004906 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004907 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4908 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4909 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4911 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4912 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4913
4914 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004915 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004916 timestamp_ms += kFrameIntervalMs;
4917 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004918 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004919 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4921 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4923 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4924 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4925 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4926
4927 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004928 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004929 timestamp_ms += kFrameIntervalMs;
4930 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004931 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004932 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004933 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004934 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07004935 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4936 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4937 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4938 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4939
Evan Shrubsole64469032020-06-11 10:45:29 +02004940 // Trigger cpu adapt up, expect no change because quality is most limited.
4941 {
4942 auto previous_sink_wants = source.sink_wants();
4943 // Store current sink wants since we expect no change ind if there is no
4944 // change then last__wants() is not updated.
4945 video_stream_encoder_->TriggerCpuUnderuse();
4946 timestamp_ms += kFrameIntervalMs;
4947 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4948 WaitForEncodedFrame(timestamp_ms);
4949 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4950 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4951 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4952 }
4953
4954 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
4955 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004956 timestamp_ms += kFrameIntervalMs;
4957 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004958 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004959 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004960 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4961 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4962 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004963 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4964 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4965 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004966
Evan Shrubsole64469032020-06-11 10:45:29 +02004967 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004968 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02004969 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004970 timestamp_ms += kFrameIntervalMs;
4971 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004972 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004973 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004974 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4975 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4976 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4977 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4978 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004979 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004980
4981 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004982 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004983 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004984 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004985 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004986
mflodmancc3d4422017-08-03 08:27:51 -07004987 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004988}
4989
mflodmancc3d4422017-08-03 08:27:51 -07004990TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004991 const int kFrameWidth = 1920;
4992 const int kFrameHeight = 1080;
4993 // 3/4 of 1920.
4994 const int kAdaptedFrameWidth = 1440;
4995 // 3/4 of 1080 rounded down to multiple of 4.
4996 const int kAdaptedFrameHeight = 808;
4997 const int kFramerate = 24;
4998
Henrik Boström381d1092020-05-12 18:49:07 +02004999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005000 DataRate::BitsPerSec(kTargetBitrateBps),
5001 DataRate::BitsPerSec(kTargetBitrateBps),
5002 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07005003 // Trigger reconfigure encoder (without resetting the entire instance).
5004 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02005005 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07005006 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
5007 video_encoder_config.number_of_streams = 1;
5008 video_encoder_config.video_stream_factory =
5009 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07005010 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005011 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005012 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07005013
5014 video_source_.set_adaptation_enabled(true);
5015
5016 video_source_.IncomingCapturedFrame(
5017 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005018 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005019
5020 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005021 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07005022 video_source_.IncomingCapturedFrame(
5023 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005024 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07005025
mflodmancc3d4422017-08-03 08:27:51 -07005026 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07005027}
5028
mflodmancc3d4422017-08-03 08:27:51 -07005029TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07005030 const int kFrameWidth = 1280;
5031 const int kFrameHeight = 720;
5032 const int kLowFps = 2;
5033 const int kHighFps = 30;
5034
Henrik Boström381d1092020-05-12 18:49:07 +02005035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005036 DataRate::BitsPerSec(kTargetBitrateBps),
5037 DataRate::BitsPerSec(kTargetBitrateBps),
5038 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005039
5040 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5041 max_framerate_ = kLowFps;
5042
5043 // Insert 2 seconds of 2fps video.
5044 for (int i = 0; i < kLowFps * 2; ++i) {
5045 video_source_.IncomingCapturedFrame(
5046 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5047 WaitForEncodedFrame(timestamp_ms);
5048 timestamp_ms += 1000 / kLowFps;
5049 }
5050
5051 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02005052 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005053 DataRate::BitsPerSec(kTargetBitrateBps),
5054 DataRate::BitsPerSec(kTargetBitrateBps),
5055 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005056 video_source_.IncomingCapturedFrame(
5057 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5058 WaitForEncodedFrame(timestamp_ms);
5059 timestamp_ms += 1000 / kLowFps;
5060
5061 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
5062
5063 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02005064 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07005065 const int kFrameIntervalMs = 1000 / kHighFps;
5066 max_framerate_ = kHighFps;
5067 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
5068 video_source_.IncomingCapturedFrame(
5069 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5070 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
5071 // be dropped if the encoder hans't been updated with the new higher target
5072 // framerate yet, causing it to overshoot the target bitrate and then
5073 // suffering the wrath of the media optimizer.
5074 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
5075 timestamp_ms += kFrameIntervalMs;
5076 }
5077
5078 // Don expect correct measurement just yet, but it should be higher than
5079 // before.
5080 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
5081
mflodmancc3d4422017-08-03 08:27:51 -07005082 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005083}
5084
mflodmancc3d4422017-08-03 08:27:51 -07005085TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07005086 const int kFrameWidth = 1280;
5087 const int kFrameHeight = 720;
5088 const int kTargetBitrateBps = 1000000;
5089
5090 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02005091 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02005092 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005093 DataRate::BitsPerSec(kTargetBitrateBps),
5094 DataRate::BitsPerSec(kTargetBitrateBps),
5095 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005096 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005097
5098 // Insert a first video frame, causes another bitrate update.
5099 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5100 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5101 video_source_.IncomingCapturedFrame(
5102 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5103 WaitForEncodedFrame(timestamp_ms);
5104
5105 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5107 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5108 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005109
5110 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005111 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005112 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005113
5114 // Bitrate observer should not be called.
5115 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5116 video_source_.IncomingCapturedFrame(
5117 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5118 ExpectDroppedFrame();
5119
mflodmancc3d4422017-08-03 08:27:51 -07005120 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005121}
ilnik6b826ef2017-06-16 06:53:48 -07005122
Niels Möller4db138e2018-04-19 09:04:13 +02005123TEST_F(VideoStreamEncoderTest,
5124 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5125 const int kFrameWidth = 1280;
5126 const int kFrameHeight = 720;
5127 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005129 DataRate::BitsPerSec(kTargetBitrateBps),
5130 DataRate::BitsPerSec(kTargetBitrateBps),
5131 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005132 video_source_.IncomingCapturedFrame(
5133 CreateFrame(1, kFrameWidth, kFrameHeight));
5134 WaitForEncodedFrame(1);
5135 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5136 .low_encode_usage_threshold_percent,
5137 default_options.low_encode_usage_threshold_percent);
5138 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5139 .high_encode_usage_threshold_percent,
5140 default_options.high_encode_usage_threshold_percent);
5141 video_stream_encoder_->Stop();
5142}
5143
5144TEST_F(VideoStreamEncoderTest,
5145 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5146 const int kFrameWidth = 1280;
5147 const int kFrameHeight = 720;
5148 CpuOveruseOptions hardware_options;
5149 hardware_options.low_encode_usage_threshold_percent = 150;
5150 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005151 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005152
Henrik Boström381d1092020-05-12 18:49:07 +02005153 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005154 DataRate::BitsPerSec(kTargetBitrateBps),
5155 DataRate::BitsPerSec(kTargetBitrateBps),
5156 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005157 video_source_.IncomingCapturedFrame(
5158 CreateFrame(1, kFrameWidth, kFrameHeight));
5159 WaitForEncodedFrame(1);
5160 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5161 .low_encode_usage_threshold_percent,
5162 hardware_options.low_encode_usage_threshold_percent);
5163 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5164 .high_encode_usage_threshold_percent,
5165 hardware_options.high_encode_usage_threshold_percent);
5166 video_stream_encoder_->Stop();
5167}
5168
Niels Möller6bb5ab92019-01-11 11:11:10 +01005169TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5170 const int kFrameWidth = 320;
5171 const int kFrameHeight = 240;
5172 const int kFps = 30;
5173 const int kTargetBitrateBps = 120000;
5174 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5175
Henrik Boström381d1092020-05-12 18:49:07 +02005176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005177 DataRate::BitsPerSec(kTargetBitrateBps),
5178 DataRate::BitsPerSec(kTargetBitrateBps),
5179 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005180
5181 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5182 max_framerate_ = kFps;
5183
5184 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5185 fake_encoder_.SimulateOvershoot(1.0);
5186 int num_dropped = 0;
5187 for (int i = 0; i < kNumFramesInRun; ++i) {
5188 video_source_.IncomingCapturedFrame(
5189 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5190 // Wait up to two frame durations for a frame to arrive.
5191 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5192 ++num_dropped;
5193 }
5194 timestamp_ms += 1000 / kFps;
5195 }
5196
Erik Språnga8d48ab2019-02-08 14:17:40 +01005197 // Framerate should be measured to be near the expected target rate.
5198 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5199
5200 // Frame drops should be within 5% of expected 0%.
5201 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005202
5203 // Make encoder produce frames at double the expected bitrate during 3 seconds
5204 // of video, verify number of drops. Rate needs to be slightly changed in
5205 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005206 double overshoot_factor = 2.0;
5207 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5208 // With bitrate adjuster, when need to overshoot even more to trigger
5209 // frame dropping.
5210 overshoot_factor *= 2;
5211 }
5212 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005213 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005214 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5215 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5216 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005217 num_dropped = 0;
5218 for (int i = 0; i < kNumFramesInRun; ++i) {
5219 video_source_.IncomingCapturedFrame(
5220 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5221 // Wait up to two frame durations for a frame to arrive.
5222 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5223 ++num_dropped;
5224 }
5225 timestamp_ms += 1000 / kFps;
5226 }
5227
Henrik Boström381d1092020-05-12 18:49:07 +02005228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005229 DataRate::BitsPerSec(kTargetBitrateBps),
5230 DataRate::BitsPerSec(kTargetBitrateBps),
5231 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005232
5233 // Target framerate should be still be near the expected target, despite
5234 // the frame drops.
5235 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5236
5237 // Frame drops should be within 5% of expected 50%.
5238 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005239
5240 video_stream_encoder_->Stop();
5241}
5242
5243TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5244 const int kFrameWidth = 320;
5245 const int kFrameHeight = 240;
5246 const int kActualInputFps = 24;
5247 const int kTargetBitrateBps = 120000;
5248
5249 ASSERT_GT(max_framerate_, kActualInputFps);
5250
5251 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5252 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005254 DataRate::BitsPerSec(kTargetBitrateBps),
5255 DataRate::BitsPerSec(kTargetBitrateBps),
5256 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005257
5258 // Insert 3 seconds of video, with an input fps lower than configured max.
5259 for (int i = 0; i < kActualInputFps * 3; ++i) {
5260 video_source_.IncomingCapturedFrame(
5261 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5262 // Wait up to two frame durations for a frame to arrive.
5263 WaitForEncodedFrame(timestamp_ms);
5264 timestamp_ms += 1000 / kActualInputFps;
5265 }
5266
5267 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5268
5269 video_stream_encoder_->Stop();
5270}
5271
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005272TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5273 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005274 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005275 DataRate::BitsPerSec(kTargetBitrateBps),
5276 DataRate::BitsPerSec(kTargetBitrateBps),
5277 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005278
5279 fake_encoder_.BlockNextEncode();
5280 video_source_.IncomingCapturedFrame(
5281 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5282 WaitForEncodedFrame(1);
5283 // On the very first frame full update should be forced.
5284 rect = fake_encoder_.GetLastUpdateRect();
5285 EXPECT_EQ(rect.offset_x, 0);
5286 EXPECT_EQ(rect.offset_y, 0);
5287 EXPECT_EQ(rect.height, codec_height_);
5288 EXPECT_EQ(rect.width, codec_width_);
5289 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5290 // call to ContinueEncode.
5291 video_source_.IncomingCapturedFrame(
5292 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5293 ExpectDroppedFrame();
5294 video_source_.IncomingCapturedFrame(
5295 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5296 ExpectDroppedFrame();
5297 fake_encoder_.ContinueEncode();
5298 WaitForEncodedFrame(3);
5299 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5300 rect = fake_encoder_.GetLastUpdateRect();
5301 EXPECT_EQ(rect.offset_x, 1);
5302 EXPECT_EQ(rect.offset_y, 0);
5303 EXPECT_EQ(rect.width, 10);
5304 EXPECT_EQ(rect.height, 1);
5305
5306 video_source_.IncomingCapturedFrame(
5307 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5308 WaitForEncodedFrame(4);
5309 // Previous frame was encoded, so no accumulation should happen.
5310 rect = fake_encoder_.GetLastUpdateRect();
5311 EXPECT_EQ(rect.offset_x, 0);
5312 EXPECT_EQ(rect.offset_y, 0);
5313 EXPECT_EQ(rect.width, 1);
5314 EXPECT_EQ(rect.height, 1);
5315
5316 video_stream_encoder_->Stop();
5317}
5318
Erik Språngd7329ca2019-02-21 21:19:53 +01005319TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005321 DataRate::BitsPerSec(kTargetBitrateBps),
5322 DataRate::BitsPerSec(kTargetBitrateBps),
5323 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005324
5325 // First frame is always keyframe.
5326 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5327 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005328 EXPECT_THAT(
5329 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005330 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005331
5332 // Insert delta frame.
5333 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5334 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005335 EXPECT_THAT(
5336 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005337 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005338
5339 // Request next frame be a key-frame.
5340 video_stream_encoder_->SendKeyFrame();
5341 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5342 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005343 EXPECT_THAT(
5344 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005345 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005346
5347 video_stream_encoder_->Stop();
5348}
5349
5350TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5351 // Setup simulcast with three streams.
5352 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005354 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5355 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5356 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005357 // Wait for all three layers before triggering event.
5358 sink_.SetNumExpectedLayers(3);
5359
5360 // First frame is always keyframe.
5361 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5362 WaitForEncodedFrame(1);
5363 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005364 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5365 VideoFrameType::kVideoFrameKey,
5366 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005367
5368 // Insert delta frame.
5369 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5370 WaitForEncodedFrame(2);
5371 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005372 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5373 VideoFrameType::kVideoFrameDelta,
5374 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005375
5376 // Request next frame be a key-frame.
5377 // Only first stream is configured to produce key-frame.
5378 video_stream_encoder_->SendKeyFrame();
5379 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5380 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005381
5382 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5383 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005384 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005385 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005386 VideoFrameType::kVideoFrameKey,
5387 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005388
5389 video_stream_encoder_->Stop();
5390}
5391
5392TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5393 // Configure internal source factory and setup test again.
5394 encoder_factory_.SetHasInternalSource(true);
5395 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005397 DataRate::BitsPerSec(kTargetBitrateBps),
5398 DataRate::BitsPerSec(kTargetBitrateBps),
5399 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005400
5401 // Call encoder directly, simulating internal source where encoded frame
5402 // callback in VideoStreamEncoder is called despite no OnFrame().
5403 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5404 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005405 EXPECT_THAT(
5406 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005407 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005408
Niels Möller8f7ce222019-03-21 15:43:58 +01005409 const std::vector<VideoFrameType> kDeltaFrame = {
5410 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005411 // Need to set timestamp manually since manually for injected frame.
5412 VideoFrame frame = CreateFrame(101, nullptr);
5413 frame.set_timestamp(101);
5414 fake_encoder_.InjectFrame(frame, false);
5415 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005416 EXPECT_THAT(
5417 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005418 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005419
5420 // Request key-frame. The forces a dummy frame down into the encoder.
5421 fake_encoder_.ExpectNullFrame();
5422 video_stream_encoder_->SendKeyFrame();
5423 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005424 EXPECT_THAT(
5425 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005426 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005427
5428 video_stream_encoder_->Stop();
5429}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005430
5431TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5432 // Configure internal source factory and setup test again.
5433 encoder_factory_.SetHasInternalSource(true);
5434 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005436 DataRate::BitsPerSec(kTargetBitrateBps),
5437 DataRate::BitsPerSec(kTargetBitrateBps),
5438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005439
5440 int64_t timestamp = 1;
5441 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005442 image.SetEncodedData(
5443 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005444 image.capture_time_ms_ = ++timestamp;
5445 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5446 const int64_t kEncodeFinishDelayMs = 10;
5447 image.timing_.encode_start_ms = timestamp;
5448 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5449 fake_encoder_.InjectEncodedImage(image);
5450 // Wait for frame without incrementing clock.
5451 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5452 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5453 // capture timestamp should be kEncodeFinishDelayMs in the past.
5454 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5455 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5456 kEncodeFinishDelayMs);
5457
5458 video_stream_encoder_->Stop();
5459}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005460
5461TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005462 // SPS contains VUI with restrictions on the maximum number of reordered
5463 // pictures, there is no need to rewrite the bitstream to enable faster
5464 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005465 ResetEncoder("H264", 1, 1, 1, false);
5466
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5468 DataRate::BitsPerSec(kTargetBitrateBps),
5469 DataRate::BitsPerSec(kTargetBitrateBps),
5470 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5471 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005472
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005473 fake_encoder_.SetEncodedImageData(
5474 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005475
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005476 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5477 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005478
5479 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5480 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005481
5482 video_stream_encoder_->Stop();
5483}
5484
5485TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005486 // SPS does not contain VUI, the bitstream is will be rewritten with added
5487 // VUI with restrictions on the maximum number of reordered pictures to
5488 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005489 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5490 0x00, 0x00, 0x03, 0x03, 0xF4,
5491 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005492 ResetEncoder("H264", 1, 1, 1, false);
5493
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005494 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5495 DataRate::BitsPerSec(kTargetBitrateBps),
5496 DataRate::BitsPerSec(kTargetBitrateBps),
5497 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5498 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005499
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005500 fake_encoder_.SetEncodedImageData(
5501 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005502
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005503 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5504 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005505
5506 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5507 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005508
5509 video_stream_encoder_->Stop();
5510}
5511
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005512TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5513 const int kFrameWidth = 1280;
5514 const int kFrameHeight = 720;
5515 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5516
Henrik Boström381d1092020-05-12 18:49:07 +02005517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005518 DataRate::BitsPerSec(kTargetBitrateBps),
5519 DataRate::BitsPerSec(kTargetBitrateBps),
5520 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005521 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5522
5523 // Insert a first video frame. It should be dropped because of downscale in
5524 // resolution.
5525 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5526 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5527 frame.set_rotation(kVideoRotation_270);
5528 video_source_.IncomingCapturedFrame(frame);
5529
5530 ExpectDroppedFrame();
5531
5532 // Second frame is downscaled.
5533 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5534 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5535 frame.set_rotation(kVideoRotation_90);
5536 video_source_.IncomingCapturedFrame(frame);
5537
5538 WaitForEncodedFrame(timestamp_ms);
5539 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5540
5541 // Insert another frame, also downscaled.
5542 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5543 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5544 frame.set_rotation(kVideoRotation_180);
5545 video_source_.IncomingCapturedFrame(frame);
5546
5547 WaitForEncodedFrame(timestamp_ms);
5548 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5549
5550 video_stream_encoder_->Stop();
5551}
5552
Erik Språng5056af02019-09-02 15:53:11 +02005553TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5554 const int kFrameWidth = 320;
5555 const int kFrameHeight = 180;
5556
5557 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005559 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5560 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5561 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005562 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005563 /*rtt_ms=*/0,
5564 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005565
5566 // Insert a first video frame so that encoder gets configured.
5567 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5568 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5569 frame.set_rotation(kVideoRotation_270);
5570 video_source_.IncomingCapturedFrame(frame);
5571 WaitForEncodedFrame(timestamp_ms);
5572
5573 // Set a target rate below the minimum allowed by the codec settings.
5574 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005575 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5576 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005578 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005579 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005580 /*link_allocation=*/target_rate,
5581 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005582 /*rtt_ms=*/0,
5583 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005584 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5585
5586 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5587 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5588 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005589 DataRate allocation_sum =
5590 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005591 EXPECT_EQ(min_rate, allocation_sum);
5592 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5593
5594 video_stream_encoder_->Stop();
5595}
5596
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005597TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005599 DataRate::BitsPerSec(kTargetBitrateBps),
5600 DataRate::BitsPerSec(kTargetBitrateBps),
5601 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005602 // Capture a frame and wait for it to synchronize with the encoder thread.
5603 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5604 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5605 WaitForEncodedFrame(1);
5606
5607 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5608 ASSERT_TRUE(prev_rate_settings.has_value());
5609 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5610 kDefaultFramerate);
5611
5612 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5613 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5614 timestamp_ms += 1000 / kDefaultFramerate;
5615 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5616 WaitForEncodedFrame(timestamp_ms);
5617 }
5618 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5619 kDefaultFramerate);
5620 // Capture larger frame to trigger a reconfigure.
5621 codec_height_ *= 2;
5622 codec_width_ *= 2;
5623 timestamp_ms += 1000 / kDefaultFramerate;
5624 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5625 WaitForEncodedFrame(timestamp_ms);
5626
5627 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5628 auto current_rate_settings =
5629 fake_encoder_.GetAndResetLastRateControlSettings();
5630 // Ensure we have actually reconfigured twice
5631 // The rate settings should have been set again even though
5632 // they haven't changed.
5633 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005634 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005635
5636 video_stream_encoder_->Stop();
5637}
5638
philipeld9cc8c02019-09-16 14:53:40 +02005639struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005640 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5641 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5642 MOCK_METHOD(void,
5643 RequestEncoderSwitch,
5644 (const webrtc::SdpVideoFormat& format),
5645 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005646};
5647
5648TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5649 constexpr int kDontCare = 100;
5650
5651 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5652 video_send_config_.encoder_settings.encoder_switch_request_callback =
5653 &switch_callback;
5654 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5655 encoder_config.codec_type = kVideoCodecVP8;
5656 webrtc::test::ScopedFieldTrials field_trial(
5657 "WebRTC-NetworkCondition-EncoderSwitch/"
5658 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5659 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5660
5661 // Reset encoder for new configuration to take effect.
5662 ConfigureEncoder(std::move(encoder_config));
5663
5664 // Send one frame to trigger ReconfigureEncoder.
5665 video_source_.IncomingCapturedFrame(
5666 CreateFrame(kDontCare, kDontCare, kDontCare));
5667
5668 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005669 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5670 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005671 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005672 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005673
Henrik Boström381d1092020-05-12 18:49:07 +02005674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005675 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5676 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5677 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005678 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005679 /*rtt_ms=*/0,
5680 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005681
5682 video_stream_encoder_->Stop();
5683}
5684
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005685TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5686 constexpr int kDontCare = 100;
5687
5688 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5689 video_send_config_.encoder_settings.encoder_switch_request_callback =
5690 &switch_callback;
5691 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5692 encoder_config.codec_type = kVideoCodecVP8;
5693 webrtc::test::ScopedFieldTrials field_trial(
5694 "WebRTC-NetworkCondition-EncoderSwitch/"
5695 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5696 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5697
5698 // Reset encoder for new configuration to take effect.
5699 ConfigureEncoder(std::move(encoder_config));
5700
5701 // Send one frame to trigger ReconfigureEncoder.
5702 video_source_.IncomingCapturedFrame(
5703 CreateFrame(kDontCare, kDontCare, kDontCare));
5704
5705 using Config = EncoderSwitchRequestCallback::Config;
5706 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5707 .Times(0);
5708
Henrik Boström381d1092020-05-12 18:49:07 +02005709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005710 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5711 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5712 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5713 /*fraction_lost=*/0,
5714 /*rtt_ms=*/0,
5715 /*cwnd_reduce_ratio=*/0);
5716
5717 video_stream_encoder_->Stop();
5718}
5719
philipeld9cc8c02019-09-16 14:53:40 +02005720TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5721 constexpr int kSufficientBitrateToNotDrop = 1000;
5722 constexpr int kHighRes = 500;
5723 constexpr int kLowRes = 100;
5724
5725 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5726 video_send_config_.encoder_settings.encoder_switch_request_callback =
5727 &switch_callback;
5728 webrtc::test::ScopedFieldTrials field_trial(
5729 "WebRTC-NetworkCondition-EncoderSwitch/"
5730 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5731 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5732 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5733 encoder_config.codec_type = kVideoCodecH264;
5734
5735 // Reset encoder for new configuration to take effect.
5736 ConfigureEncoder(std::move(encoder_config));
5737
5738 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5739 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5740 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005742 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5743 /*stable_target_bitrate=*/
5744 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5745 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005746 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005747 /*rtt_ms=*/0,
5748 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005749
5750 // Send one frame to trigger ReconfigureEncoder.
5751 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5752 WaitForEncodedFrame(1);
5753
5754 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005755 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5756 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005757 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005758 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005759
5760 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5761 WaitForEncodedFrame(2);
5762
5763 video_stream_encoder_->Stop();
5764}
5765
philipel9b058032020-02-10 11:30:00 +01005766TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5767 constexpr int kDontCare = 100;
5768 StrictMock<MockEncoderSelector> encoder_selector;
5769 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5770 &fake_encoder_, &encoder_selector);
5771 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5772
5773 // Reset encoder for new configuration to take effect.
5774 ConfigureEncoder(video_encoder_config_.Copy());
5775
5776 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5777
5778 video_source_.IncomingCapturedFrame(
5779 CreateFrame(kDontCare, kDontCare, kDontCare));
5780 video_stream_encoder_->Stop();
5781
5782 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5783 // to it's factory, so in order for the encoder instance in the
5784 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5785 // reset the |video_stream_encoder_| here.
5786 video_stream_encoder_.reset();
5787}
5788
5789TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5790 constexpr int kDontCare = 100;
5791
5792 NiceMock<MockEncoderSelector> encoder_selector;
5793 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5794 video_send_config_.encoder_settings.encoder_switch_request_callback =
5795 &switch_callback;
5796 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5797 &fake_encoder_, &encoder_selector);
5798 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5799
5800 // Reset encoder for new configuration to take effect.
5801 ConfigureEncoder(video_encoder_config_.Copy());
5802
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005803 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005804 .WillByDefault(Return(SdpVideoFormat("AV1")));
5805 EXPECT_CALL(switch_callback,
5806 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5807 Field(&SdpVideoFormat::name, "AV1"))));
5808
Henrik Boström381d1092020-05-12 18:49:07 +02005809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005810 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5811 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5812 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005813 /*fraction_lost=*/0,
5814 /*rtt_ms=*/0,
5815 /*cwnd_reduce_ratio=*/0);
5816
5817 video_stream_encoder_->Stop();
5818}
5819
5820TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5821 constexpr int kSufficientBitrateToNotDrop = 1000;
5822 constexpr int kDontCare = 100;
5823
5824 NiceMock<MockVideoEncoder> video_encoder;
5825 NiceMock<MockEncoderSelector> encoder_selector;
5826 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5827 video_send_config_.encoder_settings.encoder_switch_request_callback =
5828 &switch_callback;
5829 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5830 &video_encoder, &encoder_selector);
5831 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5832
5833 // Reset encoder for new configuration to take effect.
5834 ConfigureEncoder(video_encoder_config_.Copy());
5835
5836 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5837 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5838 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005840 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5841 /*stable_target_bitrate=*/
5842 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5843 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005844 /*fraction_lost=*/0,
5845 /*rtt_ms=*/0,
5846 /*cwnd_reduce_ratio=*/0);
5847
5848 ON_CALL(video_encoder, Encode(_, _))
5849 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5850 ON_CALL(encoder_selector, OnEncoderBroken())
5851 .WillByDefault(Return(SdpVideoFormat("AV2")));
5852
5853 rtc::Event encode_attempted;
5854 EXPECT_CALL(switch_callback,
5855 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5856 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5857 EXPECT_EQ(format.name, "AV2");
5858 encode_attempted.Set();
5859 });
5860
5861 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5862 encode_attempted.Wait(3000);
5863
5864 video_stream_encoder_->Stop();
5865
5866 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5867 // to it's factory, so in order for the encoder instance in the
5868 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5869 // reset the |video_stream_encoder_| here.
5870 video_stream_encoder_.reset();
5871}
5872
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005873TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005874 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005875 const int kFrameWidth = 320;
5876 const int kFrameHeight = 180;
5877
5878 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005879 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005881 /*target_bitrate=*/rate,
5882 /*stable_target_bitrate=*/rate,
5883 /*link_allocation=*/rate,
5884 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005885 /*rtt_ms=*/0,
5886 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005887
5888 // Insert a first video frame so that encoder gets configured.
5889 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5890 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5891 frame.set_rotation(kVideoRotation_270);
5892 video_source_.IncomingCapturedFrame(frame);
5893 WaitForEncodedFrame(timestamp_ms);
5894 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5895
5896 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005897 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005899 /*target_bitrate=*/new_stable_rate,
5900 /*stable_target_bitrate=*/new_stable_rate,
5901 /*link_allocation=*/rate,
5902 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005903 /*rtt_ms=*/0,
5904 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005905 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5906 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5907 video_stream_encoder_->Stop();
5908}
5909
5910TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005911 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005912 const int kFrameWidth = 320;
5913 const int kFrameHeight = 180;
5914
5915 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005916 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005917 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005918 /*target_bitrate=*/rate,
5919 /*stable_target_bitrate=*/rate,
5920 /*link_allocation=*/rate,
5921 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005922 /*rtt_ms=*/0,
5923 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005924
5925 // Insert a first video frame so that encoder gets configured.
5926 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5927 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5928 frame.set_rotation(kVideoRotation_270);
5929 video_source_.IncomingCapturedFrame(frame);
5930 WaitForEncodedFrame(timestamp_ms);
5931 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5932
5933 // Set a higher target rate without changing the link_allocation. Should not
5934 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005935 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005936 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005937 /*target_bitrate=*/rate,
5938 /*stable_target_bitrate=*/new_stable_rate,
5939 /*link_allocation=*/rate,
5940 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005941 /*rtt_ms=*/0,
5942 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005943 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5944 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5945 video_stream_encoder_->Stop();
5946}
5947
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005948TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5949 test::ScopedFieldTrials field_trials(
5950 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5951 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5952 const int kFramerateFps = 30;
5953 const int kWidth = 1920;
5954 const int kHeight = 1080;
5955 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5956 // Works on screenshare mode.
5957 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5958 // We rely on the automatic resolution adaptation, but we handle framerate
5959 // adaptation manually by mocking the stats proxy.
5960 video_source_.set_adaptation_enabled(true);
5961
5962 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005964 DataRate::BitsPerSec(kTargetBitrateBps),
5965 DataRate::BitsPerSec(kTargetBitrateBps),
5966 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005967 video_stream_encoder_->SetSource(&video_source_,
5968 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005969 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005970
5971 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5972 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5973
5974 // Pass enough frames with the full update to trigger animation detection.
5975 for (int i = 0; i < kNumFrames; ++i) {
5976 int64_t timestamp_ms =
5977 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5978 frame.set_ntp_time_ms(timestamp_ms);
5979 frame.set_timestamp_us(timestamp_ms * 1000);
5980 video_source_.IncomingCapturedFrame(frame);
5981 WaitForEncodedFrame(timestamp_ms);
5982 }
5983
5984 // Resolution should be limited.
5985 rtc::VideoSinkWants expected;
5986 expected.max_framerate_fps = kFramerateFps;
5987 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005988 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005989
5990 // Pass one frame with no known update.
5991 // Resolution cap should be removed immediately.
5992 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5993 frame.set_ntp_time_ms(timestamp_ms);
5994 frame.set_timestamp_us(timestamp_ms * 1000);
5995 frame.clear_update_rect();
5996
5997 video_source_.IncomingCapturedFrame(frame);
5998 WaitForEncodedFrame(timestamp_ms);
5999
6000 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006001 EXPECT_THAT(video_source_.sink_wants(),
6002 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01006003
6004 video_stream_encoder_->Stop();
6005}
6006
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02006007TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
6008 const int kWidth = 720; // 540p adapted down.
6009 const int kHeight = 405;
6010 const int kNumFrames = 3;
6011 // Works on screenshare mode.
6012 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
6013 /*num_spatial_layers=*/2, /*screenshare=*/true);
6014
6015 video_source_.set_adaptation_enabled(true);
6016
6017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6018 DataRate::BitsPerSec(kTargetBitrateBps),
6019 DataRate::BitsPerSec(kTargetBitrateBps),
6020 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6021
6022 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
6023
6024 // Pass enough frames with the full update to trigger animation detection.
6025 for (int i = 0; i < kNumFrames; ++i) {
6026 int64_t timestamp_ms =
6027 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
6028 frame.set_ntp_time_ms(timestamp_ms);
6029 frame.set_timestamp_us(timestamp_ms * 1000);
6030 video_source_.IncomingCapturedFrame(frame);
6031 WaitForEncodedFrame(timestamp_ms);
6032 }
6033
6034 video_stream_encoder_->Stop();
6035}
6036
perkj26091b12016-09-01 01:17:40 -07006037} // namespace webrtc